Compare commits
153 Commits
6721c8a224
..
low/T5
| Author | SHA1 | Date | |
|---|---|---|---|
| 41bdc178c2 | |||
| e9dc13689a | |||
| f62b017cc7 | |||
| dbcbc5bd41 | |||
| 1145df682e | |||
| 3e5a5fde81 | |||
| b62981ff80 | |||
| 31a7c8a88a | |||
| 7757b9c6fc | |||
| 0038d46c9a | |||
| cea3930b94 | |||
| f627cee9c0 | |||
| 14cb283172 | |||
| 522cfbfda6 | |||
| 1ce0c5955f | |||
| 9377fa1678 | |||
| 92246c9734 | |||
| 1fee9755d7 | |||
| 205bddb3b8 | |||
| 206c588608 | |||
| bd929d98fc | |||
| 990f4c018d | |||
| 5da4ab3ebd | |||
| 3d2f3f59b2 | |||
| 58c2e14c38 | |||
| 58f605609d | |||
| f42aa14033 | |||
| 818775b311 | |||
| 4e01cbc2c3 | |||
| 1f8ab8e3b3 | |||
| 846b77f5b4 | |||
| da276ff15a | |||
| 81e0005628 | |||
| b850c3a47f | |||
| c5e894d9b3 | |||
| 74066bbe27 | |||
| 6084465e23 | |||
| d6fe3ae62a | |||
| 90b7cc6c48 | |||
| 9b6afe3efd | |||
| cbfe296279 | |||
| 105c230cb1 | |||
| 944595ad32 | |||
| f088cb81ba | |||
| 003783a593 | |||
| 4f97e72ac9 | |||
| c582508ae5 | |||
| 8f31184b7d | |||
| 8e99b3b4c0 | |||
| 7926e8bb46 | |||
| 76098ffcd4 | |||
| 1cc4ccd258 | |||
| 20e16c934c | |||
| ec4ad984c1 | |||
| 094307e18b | |||
| baaf0a183c | |||
| dc6410dfad | |||
| 3777e50564 | |||
| 40e07aa08e | |||
| 9dd288b3ad | |||
| f313c5e62e | |||
| f9d4e2f514 | |||
| d603a2f66e | |||
| 49fbbcfaeb | |||
| e7a126a9fc | |||
| 5fc75d93fd | |||
| 3b7dbfa082 | |||
| d131416673 | |||
| f0f607ea46 | |||
| 2a38b6ac76 | |||
| 75648ea28b | |||
| 84af475683 | |||
| 3ce9cc3657 | |||
| 1c675e9153 | |||
| 92ff6a0d8c | |||
| 174ad952dc | |||
| 0a8fe68d0a | |||
| 8997a46b74 | |||
| 5fbf53601c | |||
| 0c1702acbc | |||
| 2dc919e8a4 | |||
| 95865f0e9b | |||
| 2f96709341 | |||
| 46523db2bd | |||
| 31ca9407a6 | |||
| e62b4a5baf | |||
| 7a2980e2f1 | |||
| 0d4978953f | |||
| 7df05ccc27 | |||
| 294eb163cd | |||
| aada53b27a | |||
| 5198d3dc7f | |||
| c86cead5c2 | |||
| 321ab22111 | |||
| 5f81e65684 | |||
| 3e3748b985 | |||
| 55e0178d5c | |||
| 1c31e6bcb8 | |||
| 353af72265 | |||
| c5ea3a7b59 | |||
| 7fc77f2790 | |||
| cc95590196 | |||
| cf6d19d09d | |||
| 5fd7bf524a | |||
| ad7df925ae | |||
| 591307c13e | |||
| 537a4f524d | |||
| 1a7bff6a19 | |||
| c8c4c2bcde | |||
| 35f371ed74 | |||
| 63cf4c6f0c | |||
| 0e416cac0a | |||
| 7ed2c29842 | |||
| 2b45a59c5d | |||
| d30f54639f | |||
| 0bbe42eaa1 | |||
| 6d222ea491 | |||
| 7a6f997180 | |||
| 555d1b4afc | |||
| 883601a72b | |||
| 6b12a7b75e | |||
| 0121f55f1e | |||
| c07659c0e0 | |||
| ded7cf0d6d | |||
| f7f4554a36 | |||
| 850e23646a | |||
| ff073450dc | |||
| e96b5d49f9 | |||
| b0b57f8d54 | |||
| eedbf13712 | |||
| 035c5feeb8 | |||
| b02d7a92bb | |||
| 75f267c4c9 | |||
| 7dcd93c326 | |||
| 591eaac881 | |||
| 6ab50fed1f | |||
| a58d79b793 | |||
| bf75e265ab | |||
| 8698d1804b | |||
| 344bdc4da5 | |||
| 6a4fa220de | |||
| a61e041e58 | |||
| c81a6615ba | |||
| 280e57896b | |||
| af1e0a86b2 | |||
| c22b06deba | |||
| d95292c95d | |||
| 41013dcab3 | |||
| 92997b40dc | |||
| 5fe7135e2c | |||
| 61fc88c8a8 | |||
| d0110cf406 | |||
| 80668a07bd |
@@ -1,4 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# pack.sh — produce the ZB.MOM.WW.Audit NuGet package into ./artifacts.
|
||||
set -euo pipefail
|
||||
dotnet pack -c Release -o ./artifacts
|
||||
# cd to the repo root and name the solution + an absolute output path: `dotnet pack` with
|
||||
# neither a project argument nor a fixed working directory packs whatever solution happens
|
||||
# to sit in the CALLER's directory (the bug behind the LocalDb feed incident, 0038d46).
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
dotnet pack ZB.MOM.WW.Audit.slnx -c Release -o "$REPO_ROOT/artifacts"
|
||||
|
||||
@@ -16,9 +16,20 @@ set -euo pipefail
|
||||
: "${GITEA_NUGET_SOURCE:?set GITEA_NUGET_SOURCE to your Gitea NuGet feed URL}"
|
||||
: "${GITEA_NUGET_KEY:?set GITEA_NUGET_KEY to your Gitea access token}"
|
||||
|
||||
dotnet pack -c Release -o ./artifacts
|
||||
# Named solution, named output, and a cd to the repo root, all for one reason: `dotnet pack`
|
||||
# with neither a project argument nor a fixed working directory packs whatever solution
|
||||
# happens to sit in the CALLER's directory — and then pushes it to a live feed under this
|
||||
# script's name. Invoked by absolute path from another repo's checkout, the relative form
|
||||
# packed and published 25 unrelated packages (LocalDb's copy of this script, fixed in
|
||||
# 0038d46; recovered with `dotnet nuget delete`, but only because the token happened to
|
||||
# carry delete rights). A publish script must be unambiguous about WHAT it publishes no
|
||||
# matter where it is run.
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
dotnet nuget push "./artifacts/*.nupkg" \
|
||||
dotnet pack ZB.MOM.WW.Audit.slnx -c Release -o "$REPO_ROOT/artifacts"
|
||||
|
||||
dotnet nuget push "$REPO_ROOT/artifacts/*.nupkg" \
|
||||
--source "$GITEA_NUGET_SOURCE" \
|
||||
--api-key "$GITEA_NUGET_KEY" \
|
||||
--skip-duplicate
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Version>0.1.5</Version>
|
||||
<!-- 0.2.1: NovellLdapConnection.Search now skips AD continuation referrals instead of
|
||||
letting them fail the search stage. Without it, a SearchBase at a domain root made
|
||||
every login fail as ServiceAccountBindFailed against a healthy directory. -->
|
||||
<Version>0.2.1</Version>
|
||||
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
|
||||
</PropertyGroup>
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
<PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.7" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.7" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.7" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.7" />
|
||||
|
||||
<!-- ASP.NET Core Authentication / Authorization -->
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.7" />
|
||||
|
||||
@@ -6,4 +6,10 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
dotnet pack -c Release -o ./artifacts
|
||||
# cd to the repo root and name the solution + an absolute output path: `dotnet pack` with
|
||||
# neither a project argument nor a fixed working directory packs whatever solution happens
|
||||
# to sit in the CALLER's directory (the bug behind the LocalDb feed incident, 0038d46).
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
dotnet pack ZB.MOM.WW.Auth.slnx -c Release -o "$REPO_ROOT/artifacts"
|
||||
|
||||
@@ -16,9 +16,20 @@ set -euo pipefail
|
||||
: "${GITEA_NUGET_SOURCE:?set GITEA_NUGET_SOURCE to your Gitea NuGet feed URL}"
|
||||
: "${GITEA_NUGET_KEY:?set GITEA_NUGET_KEY to your Gitea access token}"
|
||||
|
||||
dotnet pack -c Release -o ./artifacts
|
||||
# Named solution, named output, and a cd to the repo root, all for one reason: `dotnet pack`
|
||||
# with neither a project argument nor a fixed working directory packs whatever solution
|
||||
# happens to sit in the CALLER's directory — and then pushes it to a live feed under this
|
||||
# script's name. Invoked by absolute path from another repo's checkout, the relative form
|
||||
# packed and published 25 unrelated packages (LocalDb's copy of this script, fixed in
|
||||
# 0038d46; recovered with `dotnet nuget delete`, but only because the token happened to
|
||||
# carry delete rights). A publish script must be unambiguous about WHAT it publishes no
|
||||
# matter where it is run.
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
dotnet nuget push "./artifacts/*.nupkg" \
|
||||
dotnet pack ZB.MOM.WW.Auth.slnx -c Release -o "$REPO_ROOT/artifacts"
|
||||
|
||||
dotnet nuget push "$REPO_ROOT/artifacts/*.nupkg" \
|
||||
--source "$GITEA_NUGET_SOURCE" \
|
||||
--api-key "$GITEA_NUGET_KEY" \
|
||||
--skip-duplicate
|
||||
|
||||
@@ -19,6 +19,17 @@ public sealed record LdapOptions
|
||||
public string GroupAttribute { get; init; } = "memberOf";
|
||||
public int ConnectionTimeoutMs { get; init; } = 10_000;
|
||||
|
||||
/// <summary>
|
||||
/// Ordered fallback LDAP endpoints tried when the primary (and any earlier fallback)
|
||||
/// fails with a system-side error (connect/TLS, service-account bind, or search).
|
||||
/// Each entry is <c>"host"</c> (adopts <see cref="Port"/>) or <c>"host:port"</c>.
|
||||
/// Empty (the default) disables failover entirely — single-endpoint behavior is
|
||||
/// unchanged. The TLS validation policy (<see cref="ServerCertificateValidationCallback"/>
|
||||
/// / <see cref="AllowInsecure"/>) applies to every endpoint. Endpoint preference is
|
||||
/// sticky: the service keeps using the last endpoint that answered until it fails.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> FallbackServers { get; init; } = Array.Empty<string>();
|
||||
|
||||
/// <summary>
|
||||
/// Optional hook to harden (or, in dev, relax) TLS server-certificate validation for the
|
||||
/// <see cref="LdapTransport.Ldaps"/> / <see cref="LdapTransport.StartTls"/> transports. When
|
||||
|
||||
@@ -49,11 +49,16 @@ public static class ServiceCollectionExtensions
|
||||
// Fail fast at startup on a misconfigured directory rather than on first login.
|
||||
services.AddSingleton<IValidateOptions<LdapOptions>, LdapOptionsValidator>();
|
||||
|
||||
// LdapAuthService is stateless: it holds only a snapshot of LdapOptions and a stateless
|
||||
// connection factory, and opens/disposes a connection per call. It is not IDisposable.
|
||||
// Singleton is correct; TryAdd mirrors the pattern in AddZbApiKeyAuth (idempotency).
|
||||
// LdapAuthService opens/disposes a connection per call and is not IDisposable. Its only
|
||||
// mutable state is the sticky endpoint preference used by FallbackServers failover —
|
||||
// which is precisely why the instance must be shared, so a primary-DC outage is paid
|
||||
// for once rather than on every login. Singleton is correct; TryAdd mirrors the pattern
|
||||
// in AddZbApiKeyAuth (idempotency). The logger is resolved with GetService, not
|
||||
// GetRequiredService, so a host without logging configured still resolves the service.
|
||||
services.TryAddSingleton<ILdapAuthService>(sp =>
|
||||
new LdapAuthService(sp.GetRequiredService<IOptions<LdapOptions>>().Value));
|
||||
new LdapAuthService(
|
||||
sp.GetRequiredService<IOptions<LdapOptions>>().Value,
|
||||
sp.GetService<Microsoft.Extensions.Logging.ILogger<LdapAuthService>>()));
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
namespace ZB.MOM.WW.Auth.Ldap.Internal;
|
||||
|
||||
/// <summary>A single LDAP server endpoint resolved from configuration.</summary>
|
||||
internal sealed record LdapEndpoint(string Host, int Port)
|
||||
{
|
||||
/// <summary>Renders the endpoint as <c>host:port</c> — the form used in failover log messages.</summary>
|
||||
public override string ToString() => $"{Host}:{Port}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses <c>FallbackServers</c> entries: <c>"host"</c> (adopts the caller's default port)
|
||||
/// or <c>"host:port"</c>. IPv6 literals are not supported — use a DNS name.
|
||||
/// </summary>
|
||||
internal static class LdapEndpointParser
|
||||
{
|
||||
/// <summary>
|
||||
/// Attempts to parse a single <c>FallbackServers</c> entry.
|
||||
/// </summary>
|
||||
/// <param name="entry">The configured entry, <c>"host"</c> or <c>"host:port"</c>. Surrounding whitespace is tolerated.</param>
|
||||
/// <param name="defaultPort">Port adopted when the entry carries no explicit one.</param>
|
||||
/// <param name="endpoint">The parsed endpoint on success; undefined otherwise.</param>
|
||||
/// <param name="error">A human-readable reason on failure; empty on success.</param>
|
||||
/// <returns><see langword="true"/> when the entry parsed.</returns>
|
||||
public static bool TryParse(string entry, int defaultPort, out LdapEndpoint endpoint, out string error)
|
||||
{
|
||||
endpoint = null!;
|
||||
error = "";
|
||||
|
||||
if (string.IsNullOrWhiteSpace(entry))
|
||||
{
|
||||
error = "entry is empty or whitespace";
|
||||
return false;
|
||||
}
|
||||
|
||||
var parts = entry.Trim().Split(':');
|
||||
if (parts.Length > 2)
|
||||
{
|
||||
error = $"entry \"{entry.Trim()}\" contains more than one ':' " +
|
||||
"(IPv6 literals are not supported — use a DNS name)";
|
||||
return false;
|
||||
}
|
||||
|
||||
var host = parts[0].Trim();
|
||||
if (host.Length == 0)
|
||||
{
|
||||
error = $"entry \"{entry.Trim()}\" has an empty host";
|
||||
return false;
|
||||
}
|
||||
|
||||
var port = defaultPort;
|
||||
if (parts.Length == 2 && (!int.TryParse(parts[1].Trim(), out port) || port is < 1 or > 65535))
|
||||
{
|
||||
error = $"entry \"{entry.Trim()}\" has an invalid port (must be 1–65535)";
|
||||
return false;
|
||||
}
|
||||
|
||||
endpoint = new LdapEndpoint(host, port);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -99,23 +99,73 @@ internal sealed class NovellLdapConnection : ILdapConnection
|
||||
attributes.ToArray(),
|
||||
typesOnly: false);
|
||||
|
||||
return ReadEntries(results);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drains a result set into entries, <b>skipping continuation referrals</b>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Active Directory answers any search based at a domain root with one or more
|
||||
/// continuation references (to <c>CN=Configuration</c>, <c>DC=ForestDnsZones</c> and
|
||||
/// friends). With referral following disabled — the default, and what we want, since
|
||||
/// chasing a referral would re-bind somewhere we did not authenticate — Novell surfaces
|
||||
/// each one by throwing <see cref="LdapReferralException"/> from <see cref="ILdapSearchResults.Next"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A continuation reference is a <em>pointer to another naming context</em>, not an error:
|
||||
/// it says "there may be more results over there", never that this search failed. Letting
|
||||
/// it escape made the whole search stage fail, which <c>LdapAuthService</c> maps to
|
||||
/// <see cref="Abstractions.Ldap.LdapAuthFailure.ServiceAccountBindFailed"/> — so against a
|
||||
/// real AD domain root, every login failed with a system-side error while the directory,
|
||||
/// the credentials and both DCs were perfectly healthy. Verified against
|
||||
/// <c>DC=nam,DC=zimmer,DC=com</c> on 2026-08-12.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Skipping is the correct behaviour rather than a workaround: the entries we can see are
|
||||
/// exactly the ones in the naming context we bound to, which is the set the caller asked
|
||||
/// about. Internal so the referral handling can be unit-tested through the
|
||||
/// <see cref="ILdapSearchResults"/> seam without a live directory.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static List<LdapSearchEntry> ReadEntries(ILdapSearchResults results)
|
||||
{
|
||||
var entries = new List<LdapSearchEntry>();
|
||||
while (results.HasMore())
|
||||
var referrals = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
var entry = results.Next();
|
||||
var attrs = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (LdapAttribute attr in entry.GetAttributeSet())
|
||||
try
|
||||
{
|
||||
attrs[attr.Name] = attr.StringValueArray.ToList();
|
||||
}
|
||||
if (!results.HasMore()) break;
|
||||
|
||||
entries.Add(new LdapSearchEntry(entry.Dn, attrs));
|
||||
var entry = results.Next();
|
||||
var attrs = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (LdapAttribute attr in entry.GetAttributeSet())
|
||||
{
|
||||
attrs[attr.Name] = attr.StringValueArray.ToList();
|
||||
}
|
||||
|
||||
entries.Add(new LdapSearchEntry(entry.Dn, attrs));
|
||||
}
|
||||
catch (LdapException ex) when (ex is LdapReferralException || ex.ResultCode == LdapException.Referral)
|
||||
{
|
||||
// Novell dequeues the reference before throwing, so the enumeration advances and
|
||||
// this terminates. The cap is a belt-and-braces guard: a client that threw without
|
||||
// advancing would otherwise spin forever inside a fail-closed auth path, which is a
|
||||
// far worse failure than under-reporting an implausible number of referrals.
|
||||
if (++referrals > MaxReferrals) break;
|
||||
}
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
/// <summary>Upper bound on continuation references tolerated in one search — see <see cref="ReadEntries"/>.</summary>
|
||||
private const int MaxReferrals = 1_000;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Dispose()
|
||||
{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Novell.Directory.Ldap;
|
||||
using ZB.MOM.WW.Auth.Abstractions.Ldap;
|
||||
using ZB.MOM.WW.Auth.Ldap.Internal;
|
||||
@@ -11,6 +12,7 @@ namespace ZB.MOM.WW.Auth.Ldap;
|
||||
/// <see cref="ILdapConnection"/> so the logic is unit-testable without a live server.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Fully fail-closed: authentication never throws to the caller — every error, expected
|
||||
/// or unexpected, is mapped to a structured <see cref="LdapAuthResult.Fail(LdapAuthFailure)"/>.
|
||||
/// A success is only returned for a user that resolved to exactly one entry, whose password
|
||||
@@ -18,11 +20,38 @@ namespace ZB.MOM.WW.Auth.Ldap;
|
||||
/// Service-account bind failures (<see cref="LdapAuthFailure.ServiceAccountBindFailed"/>) are
|
||||
/// kept distinct from end-user bind failures (<see cref="LdapAuthFailure.BadCredentials"/>) so
|
||||
/// a system misconfiguration is not mistaken for bad user input.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// When <see cref="LdapOptions.FallbackServers"/> is non-empty the flow above runs against
|
||||
/// an ordered endpoint list (primary first, then fallbacks in declared order) and moves on
|
||||
/// to the next endpoint ONLY on a system-side failure — connect/TLS, service-account bind,
|
||||
/// or search — i.e. when the endpoint gave no authoritative answer. Any answer from a healthy
|
||||
/// directory (success, or a user-side failure such as
|
||||
/// <see cref="LdapAuthFailure.BadCredentials"/> / <see cref="LdapAuthFailure.UserNotFound"/>)
|
||||
/// is final and is never retried elsewhere: retrying a rejected password against a second DC
|
||||
/// would turn one bad-password event into several, tripping account lockouts. Endpoint
|
||||
/// preference is sticky — the last endpoint that answered is tried first next time — so a
|
||||
/// primary outage costs its connect timeout once, not once per login.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The service is safe to share (it is registered as a singleton). The sticky index is the
|
||||
/// only mutable state; concurrent logins may race on it, which is benign — the worst outcome
|
||||
/// is one login starting at a stale endpoint and walking on.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class LdapAuthService : ILdapAuthService
|
||||
{
|
||||
private readonly LdapOptions _options;
|
||||
private readonly ILdapConnectionFactory _connectionFactory;
|
||||
private readonly IReadOnlyList<LdapEndpoint> _endpoints;
|
||||
private readonly ILogger<LdapAuthService>? _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Index into <see cref="_endpoints"/> of the endpoint tried first. Written on any
|
||||
/// authoritative answer from a non-preferred endpoint. Accessed via
|
||||
/// <see cref="Volatile"/> only; benign races are accepted (see the type remarks).
|
||||
/// </summary>
|
||||
private int _preferredIndex;
|
||||
|
||||
/// <summary>
|
||||
/// Production constructor: binds against a live directory via the real
|
||||
@@ -31,9 +60,15 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
/// <see cref="LdapOptions.AllowInsecure"/> bypass) is carried into the factory so each
|
||||
/// connection is built with it.
|
||||
/// </summary>
|
||||
public LdapAuthService(LdapOptions options)
|
||||
/// <param name="options">The directory connection and search settings.</param>
|
||||
/// <param name="logger">
|
||||
/// Optional. When supplied, endpoint failover is made visible: a Warning per failed
|
||||
/// endpoint hop and an Information line when the sticky preference moves. Omitting it
|
||||
/// (the default) leaves the service silent — hosts without logging still work.
|
||||
/// </param>
|
||||
public LdapAuthService(LdapOptions options, ILogger<LdapAuthService>? logger = null)
|
||||
: this(options, new NovellLdapConnectionFactory(
|
||||
options.AllowInsecure, options.ServerCertificateValidationCallback))
|
||||
options.AllowInsecure, options.ServerCertificateValidationCallback), logger)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -42,10 +77,33 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
/// so the bind/search logic can be exercised without a live directory. Internal
|
||||
/// because the connection seam is an implementation detail.
|
||||
/// </summary>
|
||||
internal LdapAuthService(LdapOptions options, ILdapConnectionFactory connectionFactory)
|
||||
internal LdapAuthService(
|
||||
LdapOptions options,
|
||||
ILdapConnectionFactory connectionFactory,
|
||||
ILogger<LdapAuthService>? logger = null)
|
||||
{
|
||||
_options = options;
|
||||
_connectionFactory = connectionFactory;
|
||||
_logger = logger;
|
||||
_endpoints = BuildEndpoints(options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Primary first, then fallbacks in declared order. The validator rejects malformed
|
||||
/// entries at boot; unparseable entries are skipped defensively here so a service
|
||||
/// constructed without options validation degrades to the parseable subset rather
|
||||
/// than throwing from a constructor.
|
||||
/// </summary>
|
||||
private static IReadOnlyList<LdapEndpoint> BuildEndpoints(LdapOptions options)
|
||||
{
|
||||
var endpoints = new List<LdapEndpoint> { new(options.Server, options.Port) };
|
||||
foreach (var entry in options.FallbackServers)
|
||||
{
|
||||
if (LdapEndpointParser.TryParse(entry, options.Port, out var endpoint, out _))
|
||||
endpoints.Add(endpoint);
|
||||
}
|
||||
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -54,7 +112,10 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
/// bind-then-search-then-bind flow is wrapped so that any error — expected or unexpected —
|
||||
/// is mapped to a structured <see cref="LdapAuthResult.Fail(LdapAuthFailure)"/>. A
|
||||
/// <c>Succeeded == true</c> result is only ever returned when the user resolved to exactly
|
||||
/// one entry, their password verified, AND at least one group was extracted.
|
||||
/// one entry, their password verified, AND at least one group was extracted. With
|
||||
/// <see cref="LdapOptions.FallbackServers"/> configured the flow may run against more than
|
||||
/// one endpoint before answering — see the type-level remarks for the walk and its
|
||||
/// system-side-only failover rule.
|
||||
/// </remarks>
|
||||
public Task<LdapAuthResult> AuthenticateAsync(string username, string password, CancellationToken ct)
|
||||
{
|
||||
@@ -80,25 +141,90 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
// filter and the returned result (avoids two identities for one person).
|
||||
username = username.Trim();
|
||||
|
||||
// 4. Walk the endpoint list, preferred one first. A single-endpoint configuration
|
||||
// (no FallbackServers) collapses to exactly one attempt — unchanged behaviour.
|
||||
var preferred = Volatile.Read(ref _preferredIndex);
|
||||
if ((uint)preferred >= (uint)_endpoints.Count)
|
||||
preferred = 0; // defensive: the list is immutable, but never index out of it
|
||||
|
||||
// Seeded so an exhausted walk still returns the conservative system-side bucket
|
||||
// even in the impossible case of an empty endpoint list.
|
||||
var lastSystemSideFailure = LdapAuthResult.Fail(LdapAuthFailure.ServiceAccountBindFailed);
|
||||
|
||||
foreach (var index in WalkOrder(_endpoints.Count, preferred))
|
||||
{
|
||||
var result = AuthenticateAgainst(_endpoints[index], username, password);
|
||||
|
||||
// ServiceAccountBindFailed is the (sole) system-side bucket: this endpoint gave
|
||||
// no authoritative answer, so try the next one. Every other outcome — success, or
|
||||
// a user-side failure like UserNotFound/BadCredentials — came from a healthy
|
||||
// directory and is FINAL: re-asking a second DC about a rejected password would
|
||||
// multiply one bad-password event into several against the account lockout policy.
|
||||
if (result.Failure == LdapAuthFailure.ServiceAccountBindFailed)
|
||||
{
|
||||
lastSystemSideFailure = result;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Sticky preference: any authoritative answer proves this endpoint healthy.
|
||||
// Updating only on Succeeded would re-pay the dead primary's connect timeout on
|
||||
// every mistyped password for the whole outage.
|
||||
if (index != preferred)
|
||||
{
|
||||
Volatile.Write(ref _preferredIndex, index);
|
||||
_logger?.LogInformation("LDAP sticky preference moved to {Endpoint}.", _endpoints[index]);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// Every endpoint failed system-side. The preference is deliberately left where it was:
|
||||
// a total outage says nothing about which endpoint to prefer once one recovers.
|
||||
return lastSystemSideFailure;
|
||||
}
|
||||
|
||||
/// <summary>Preferred endpoint first, then every other endpoint in declared order.</summary>
|
||||
private static IEnumerable<int> WalkOrder(int count, int preferred)
|
||||
{
|
||||
yield return preferred;
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
if (i != preferred)
|
||||
yield return i;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs one full bind-then-search-then-bind attempt against a single endpoint on a fresh
|
||||
/// connection. The fail-closed contract holds per attempt: this never throws, and it
|
||||
/// returns <see cref="LdapAuthFailure.ServiceAccountBindFailed"/> for — and only for —
|
||||
/// system-side faults, which is what the caller reads as "no authoritative answer, try
|
||||
/// the next endpoint".
|
||||
/// </summary>
|
||||
private LdapAuthResult AuthenticateAgainst(LdapEndpoint endpoint, string username, string password)
|
||||
{
|
||||
// The whole flow runs inside an outer fail-closed guard: a StageFailure carries an
|
||||
// already-mapped failure out of a stage, and any OTHER unexpected exception defaults to
|
||||
// the most conservative system-side bucket. Either way the caller gets a structured result.
|
||||
try
|
||||
{
|
||||
// A fresh connection per attempt: a Novell connection that has failed to connect
|
||||
// or bind is not safely reusable against another host.
|
||||
using var conn = _connectionFactory.Create();
|
||||
|
||||
// 4. Open the connection (transport/TLS handling lives in the adapter). The
|
||||
// 5. Open the connection (transport/TLS handling lives in the adapter). The
|
||||
// per-operation timeout (ConnectionTimeoutMs) is applied by the adapter here.
|
||||
// A failure to connect/upgrade means the directory is unreachable — a
|
||||
// system-side fault, not the user's, so map it to ServiceAccountBindFailed.
|
||||
// A failure to connect/upgrade means this endpoint is unreachable — a
|
||||
// system-side fault, not the user's, so map it to ServiceAccountBindFailed
|
||||
// and let the caller try the next endpoint.
|
||||
// NOTE: the LdapAuthFailure enum has no dedicated DirectoryUnavailable value;
|
||||
// ServiceAccountBindFailed is the closest system-side bucket. A future
|
||||
// Abstractions change could add DirectoryUnavailable to disambiguate.
|
||||
try
|
||||
{
|
||||
conn.Connect(
|
||||
_options.Server,
|
||||
_options.Port,
|
||||
endpoint.Host,
|
||||
endpoint.Port,
|
||||
_options.Transport,
|
||||
_options.AllowInsecure,
|
||||
_options.ConnectionTimeoutMs,
|
||||
@@ -106,10 +232,10 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
}
|
||||
catch (LdapException)
|
||||
{
|
||||
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed);
|
||||
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed, "connect");
|
||||
}
|
||||
|
||||
// 5. Service-account bind so we can search for the user's DN. A bind failure
|
||||
// 6. Service-account bind so we can search for the user's DN. A bind failure
|
||||
// here is a service-account misconfiguration — DISTINCT from a user-credential
|
||||
// failure — so it maps to ServiceAccountBindFailed.
|
||||
try
|
||||
@@ -118,10 +244,10 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
}
|
||||
catch (LdapException)
|
||||
{
|
||||
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed);
|
||||
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed, "service-bind");
|
||||
}
|
||||
|
||||
// 6. Search for the user entry by the configured username attribute. A search
|
||||
// 7. Search for the user entry by the configured username attribute. A search
|
||||
// failure is infrastructure (directory unreachable / unhealthy) — system-side.
|
||||
// NOTE: same enum limitation as Connect above; ServiceAccountBindFailed is the
|
||||
// closest system-side bucket until a DirectoryUnavailable value exists.
|
||||
@@ -133,10 +259,11 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
}
|
||||
catch (LdapException)
|
||||
{
|
||||
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed);
|
||||
throw new StageFailure(LdapAuthFailure.ServiceAccountBindFailed, "search");
|
||||
}
|
||||
|
||||
// 7. Require exactly one match. Zero -> UserNotFound; two or more -> AmbiguousUser.
|
||||
// 8. Require exactly one match. Zero -> UserNotFound; two or more -> AmbiguousUser.
|
||||
// Both are authoritative answers from a healthy directory, so they end the walk.
|
||||
// We never attempt a user bind against an ambiguous DN.
|
||||
if (entries.Count == 0)
|
||||
return LdapAuthResult.Fail(LdapAuthFailure.UserNotFound);
|
||||
@@ -145,19 +272,21 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
|
||||
var entry = entries[0];
|
||||
|
||||
// 8. User bind: re-bind as the resolved DN to verify the password. A bind failure
|
||||
// here is the end user's bad credentials.
|
||||
// 9. User bind: re-bind as the resolved DN to verify the password. A bind failure
|
||||
// here is the end user's bad credentials — the directory answered, so this is
|
||||
// returned directly and never retried against another endpoint.
|
||||
try
|
||||
{
|
||||
conn.Bind(entry.Dn, password);
|
||||
}
|
||||
catch (LdapException)
|
||||
{
|
||||
throw new StageFailure(LdapAuthFailure.BadCredentials);
|
||||
return LdapAuthResult.Fail(LdapAuthFailure.BadCredentials);
|
||||
}
|
||||
|
||||
// 9. Group extraction. Fail closed: an empty/missing group set, or any error while
|
||||
// extracting groups, is a GroupLookupFailed — never a zero-group success.
|
||||
// 10. Group extraction. Fail closed: an empty/missing group set, or any error while
|
||||
// extracting groups, is a GroupLookupFailed — never a zero-group success. This
|
||||
// too is an answer about the user, not about the endpoint, so it ends the walk.
|
||||
IReadOnlyList<string> groups;
|
||||
try
|
||||
{
|
||||
@@ -165,25 +294,31 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
}
|
||||
catch
|
||||
{
|
||||
throw new StageFailure(LdapAuthFailure.GroupLookupFailed);
|
||||
return LdapAuthResult.Fail(LdapAuthFailure.GroupLookupFailed);
|
||||
}
|
||||
|
||||
if (groups.Count == 0)
|
||||
return LdapAuthResult.Fail(LdapAuthFailure.GroupLookupFailed);
|
||||
|
||||
// 10. Success — and only here, with a verified password and >= 1 group.
|
||||
// 11. Success — and only here, with a verified password and >= 1 group.
|
||||
var displayName = ExtractDisplayName(entry, username);
|
||||
return LdapAuthResult.Success(username, displayName, groups);
|
||||
}
|
||||
catch (StageFailure stage)
|
||||
{
|
||||
// A stage mapped its own failure; surface it as a structured result.
|
||||
// A stage mapped its own failure; surface it as a structured result. Log at Warning
|
||||
// so a silent failover — the failure mode this feature exists to survive — is
|
||||
// visible in the host's log rather than only in a latency graph.
|
||||
_logger?.LogWarning(
|
||||
"LDAP endpoint {Endpoint} failed system-side at {Stage}; treating as no answer.",
|
||||
endpoint, stage.Stage);
|
||||
return LdapAuthResult.Fail(stage.Failure);
|
||||
}
|
||||
catch
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Belt-and-braces: ANY unexpected exception fails closed to the most conservative
|
||||
// system-side bucket rather than propagating to the caller.
|
||||
_logger?.LogWarning(ex, "LDAP endpoint {Endpoint} failed system-side with an unexpected error.", endpoint);
|
||||
return LdapAuthResult.Fail(LdapAuthFailure.ServiceAccountBindFailed);
|
||||
}
|
||||
}
|
||||
@@ -194,9 +329,16 @@ public sealed class LdapAuthService : ILdapAuthService
|
||||
/// </summary>
|
||||
private sealed class StageFailure : Exception
|
||||
{
|
||||
public StageFailure(LdapAuthFailure failure) => Failure = failure;
|
||||
public StageFailure(LdapAuthFailure failure, string stage)
|
||||
{
|
||||
Failure = failure;
|
||||
Stage = stage;
|
||||
}
|
||||
|
||||
public LdapAuthFailure Failure { get; }
|
||||
|
||||
/// <summary>Which stage faulted (<c>connect</c> / <c>service-bind</c> / <c>search</c>), for diagnostics.</summary>
|
||||
public string Stage { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -10,7 +10,7 @@ namespace ZB.MOM.WW.Auth.Ldap;
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Validation is skipped entirely when <see cref="LdapOptions.Enabled"/> is <c>false</c>
|
||||
/// (a disabled provider's connection fields are inert). When enabled, four conditions
|
||||
/// (a disabled provider's connection fields are inert). When enabled, five conditions
|
||||
/// are enforced:
|
||||
/// <list type="bullet">
|
||||
/// <item>plaintext transport (<see cref="LdapTransport.None"/>) is rejected unless
|
||||
@@ -19,7 +19,10 @@ namespace ZB.MOM.WW.Auth.Ldap;
|
||||
/// <item><see cref="LdapOptions.SearchBase"/> must be specified (the DN root every
|
||||
/// search runs against);</item>
|
||||
/// <item><see cref="LdapOptions.ServiceAccountDn"/> must be specified — an empty value
|
||||
/// would bind anonymously, defeating the search-then-bind authentication flow.</item>
|
||||
/// would bind anonymously, defeating the search-then-bind authentication flow;</item>
|
||||
/// <item>every <see cref="LdapOptions.FallbackServers"/> entry must parse as
|
||||
/// <c>"host"</c> or <c>"host:port"</c> — a typo'd backup DC must not lie dormant
|
||||
/// until the primary outage it exists to survive.</item>
|
||||
/// </list>
|
||||
/// </remarks>
|
||||
public sealed class LdapOptionsValidator : IValidateOptions<LdapOptions>
|
||||
@@ -68,6 +71,21 @@ public sealed class LdapOptionsValidator : IValidateOptions<LdapOptions>
|
||||
"(e.g. \"cn=svc,dc=example,dc=com\").");
|
||||
}
|
||||
|
||||
// Fallback endpoints are walked by LdapAuthService on system-side failures. A
|
||||
// malformed entry would be silently skipped there (the walk degrades to the
|
||||
// parseable subset rather than throwing), so it must be caught at boot instead —
|
||||
// otherwise an operator's typo'd backup DC would only surface as a failed
|
||||
// failover during the primary outage it was meant to survive.
|
||||
foreach (var entry in options.FallbackServers)
|
||||
{
|
||||
if (!Internal.LdapEndpointParser.TryParse(entry, options.Port, out _, out var error))
|
||||
{
|
||||
return ValidateOptionsResult.Fail(
|
||||
$"{nameof(LdapOptions.FallbackServers)} contains an invalid entry — {error}. " +
|
||||
"Each entry must be \"host\" or \"host:port\" (port 1–65535).");
|
||||
}
|
||||
}
|
||||
|
||||
return ValidateOptionsResult.Success;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Novell.Directory.Ldap.NETStandard" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -99,4 +99,70 @@ public sealed class GLAuthIntegrationTests
|
||||
$"Authentication failed: {result.Failure} (server={server}:{port}, user={user})");
|
||||
Assert.NotEmpty(result.Groups);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Failover: primary endpoint points at a closed local port (refused immediately),
|
||||
/// fallback at the real GLAuth. The login must succeed via the fallback, and a second
|
||||
/// login must also succeed (sticky preference — no re-walk through the dead primary).
|
||||
/// Skipped unless <c>ZB_LDAP_IT=1</c> is set; skipped again if the server is unreachable.
|
||||
/// </summary>
|
||||
[SkippableFact]
|
||||
public async Task Authenticate_FailsOverFromDeadPrimary_ToRealGLAuth()
|
||||
{
|
||||
// ------------------------------------------------------------------ opt-in gate
|
||||
Skip.IfNot(
|
||||
Environment.GetEnvironmentVariable("ZB_LDAP_IT") == "1",
|
||||
"Set ZB_LDAP_IT=1 and a reachable GLAuth to run.");
|
||||
|
||||
// ------------------------------------------------------------------ read config
|
||||
var server = Environment.GetEnvironmentVariable("ZB_LDAP_SERVER") ?? "localhost";
|
||||
var port = int.TryParse(Environment.GetEnvironmentVariable("ZB_LDAP_PORT"), out var p) ? p : 3893;
|
||||
var baseDn = Environment.GetEnvironmentVariable("ZB_LDAP_BASE") ?? "dc=lmxopcua,dc=local";
|
||||
var svcDn = Environment.GetEnvironmentVariable("ZB_LDAP_SVC_DN") ?? "";
|
||||
var svcPw = Environment.GetEnvironmentVariable("ZB_LDAP_SVC_PW") ?? "";
|
||||
var user = Environment.GetEnvironmentVariable("ZB_LDAP_USER") ?? "";
|
||||
var pw = Environment.GetEnvironmentVariable("ZB_LDAP_PW") ?? "";
|
||||
var userAttr = Environment.GetEnvironmentVariable("ZB_LDAP_USERATTR") ?? "cn";
|
||||
|
||||
// ------------------------------------------------------------------ reachability probe
|
||||
try
|
||||
{
|
||||
using var tcp = new TcpClient();
|
||||
// 3-second connect timeout to keep the test suite snappy when the server is absent
|
||||
var connectTask = tcp.ConnectAsync(server, port);
|
||||
if (!connectTask.Wait(TimeSpan.FromSeconds(3)))
|
||||
Skip.If(true, $"GLAuth not reachable at {server}:{port} (connect timed out).");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Skip.If(true, $"GLAuth not reachable at {server}:{port}: {ex.Message}");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ build options
|
||||
var options = new LdapOptions
|
||||
{
|
||||
Enabled = true,
|
||||
Server = "127.0.0.1",
|
||||
Port = 39999, // closed port -> immediate refusal
|
||||
Transport = LdapTransport.None,
|
||||
AllowInsecure = true,
|
||||
SearchBase = baseDn,
|
||||
ServiceAccountDn = svcDn,
|
||||
ServiceAccountPassword = svcPw,
|
||||
UserNameAttribute = userAttr,
|
||||
GroupAttribute = "memberOf",
|
||||
ConnectionTimeoutMs = 3000,
|
||||
FallbackServers = new[] { $"{server}:{port}" },
|
||||
};
|
||||
|
||||
// ------------------------------------------------------------------ exercise the real service
|
||||
var svc = new LdapAuthService(options);
|
||||
|
||||
var first = await svc.AuthenticateAsync(user, pw, CancellationToken.None);
|
||||
Assert.True(first.Succeeded, $"first login failed: {first.Failure}");
|
||||
|
||||
// Second login exercises the sticky preference: it must not re-walk the dead primary.
|
||||
var second = await svc.AuthenticateAsync(user, pw, CancellationToken.None);
|
||||
Assert.True(second.Succeeded, $"second login failed: {second.Failure}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
using ZB.MOM.WW.Auth.Abstractions.Ldap;
|
||||
|
||||
namespace ZB.MOM.WW.Auth.Ldap.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Covers the multi-endpoint failover walk: which failures fail over (system-side only),
|
||||
/// which are final (any authoritative answer from a healthy directory), how the sticky
|
||||
/// endpoint preference moves, and that single-endpoint behaviour is unchanged.
|
||||
/// </summary>
|
||||
public class LdapAuthServiceFailoverTests
|
||||
{
|
||||
private static LdapOptions Opts(params string[] fallbacks) => new()
|
||||
{
|
||||
Enabled = true,
|
||||
Server = "dc1",
|
||||
Port = 636,
|
||||
Transport = LdapTransport.Ldaps,
|
||||
SearchBase = "dc=x",
|
||||
ServiceAccountDn = "cn=svc,dc=x",
|
||||
ServiceAccountPassword = "svcpw",
|
||||
FallbackServers = fallbacks,
|
||||
};
|
||||
|
||||
private static FakeLdapConnection HealthyWithUser() =>
|
||||
new FakeLdapConnection().WithUserEntry("cn=alice,dc=x", new[] { "cn=ops,dc=x" });
|
||||
|
||||
// ---- failover triggers (system-side) ----
|
||||
|
||||
[Fact]
|
||||
public async Task ConnectFailure_FailsOverToFallback_AndSucceeds()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnConnect(), HealthyWithUser());
|
||||
var svc = new LdapAuthService(Opts("dc2:3894"), factory);
|
||||
|
||||
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.True(result.Succeeded);
|
||||
Assert.Equal(2, factory.Created.Count);
|
||||
Assert.Equal("dc1", factory.Created[0].ConnectArgs!.Value.Host);
|
||||
Assert.Equal("dc2", factory.Created[1].ConnectArgs!.Value.Host);
|
||||
Assert.Equal(3894, factory.Created[1].ConnectArgs!.Value.Port);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ServiceBindFailure_FailsOverToFallback()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnServiceBind(), HealthyWithUser());
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.True(result.Succeeded);
|
||||
Assert.Equal(636, factory.Created[1].ConnectArgs!.Value.Port); // bare host adopts Port
|
||||
}
|
||||
|
||||
// ---- user-side results NEVER fail over ----
|
||||
|
||||
[Fact]
|
||||
public async Task BadUserCredentials_DoesNotFailOver()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
HealthyWithUser().ThrowOnUserBind());
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
var result = await svc.AuthenticateAsync("alice", "wrong", CancellationToken.None);
|
||||
|
||||
Assert.Equal(LdapAuthFailure.BadCredentials, result.Failure);
|
||||
Assert.Single(factory.Created);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UserNotFound_DoesNotFailOver()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(new FakeLdapConnection());
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
var result = await svc.AuthenticateAsync("ghost", "pw", CancellationToken.None);
|
||||
|
||||
Assert.Equal(LdapAuthFailure.UserNotFound, result.Failure);
|
||||
Assert.Single(factory.Created);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GroupLookupFailed_DoesNotFailOver()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().WithUserEntry("cn=alice,dc=x", Array.Empty<string>()));
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.Equal(LdapAuthFailure.GroupLookupFailed, result.Failure);
|
||||
Assert.Single(factory.Created);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AmbiguousUser_DoesNotFailOver()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().WithDuplicateMatch());
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.Equal(LdapAuthFailure.AmbiguousUser, result.Failure);
|
||||
Assert.Single(factory.Created);
|
||||
}
|
||||
|
||||
// ---- exhaustion ----
|
||||
|
||||
[Fact]
|
||||
public async Task AllEndpointsDown_ReturnsServiceAccountBindFailed()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnConnect(),
|
||||
new FakeLdapConnection().ThrowOnConnect());
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.Equal(LdapAuthFailure.ServiceAccountBindFailed, result.Failure);
|
||||
Assert.Equal(2, factory.Created.Count);
|
||||
}
|
||||
|
||||
// ---- stickiness ----
|
||||
|
||||
[Fact]
|
||||
public async Task StickyPreference_SecondLoginStartsAtLastGoodEndpoint()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1: dead
|
||||
HealthyWithUser(), // login 1 @ dc2: answers
|
||||
HealthyWithUser()); // login 2: should go straight to dc2
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
|
||||
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
|
||||
|
||||
Assert.Equal(3, factory.Created.Count);
|
||||
Assert.Equal("dc2", factory.Created[2].ConnectArgs!.Value.Host);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StickyPreference_DriftsBackToPrimary_WhenPreferredFails()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1: dead
|
||||
HealthyWithUser(), // login 1 @ dc2: answers -> preferred
|
||||
new FakeLdapConnection().ThrowOnConnect(), // login 2 @ dc2: now dead
|
||||
HealthyWithUser(), // login 2 @ dc1 (declared order): recovered
|
||||
HealthyWithUser()); // login 3: should start at dc1 again
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
|
||||
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
|
||||
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
|
||||
|
||||
Assert.Equal(5, factory.Created.Count);
|
||||
Assert.Equal("dc1", factory.Created[3].ConnectArgs!.Value.Host);
|
||||
Assert.Equal("dc1", factory.Created[4].ConnectArgs!.Value.Host);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StickyPreference_UpdatesOnUserSideAnswer_NotOnlySuccess()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1: dead
|
||||
new FakeLdapConnection(), // login 1 @ dc2: answers UserNotFound
|
||||
HealthyWithUser()); // login 2: should start at dc2
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
var first = await svc.AuthenticateAsync("ghost", "pw", CancellationToken.None);
|
||||
Assert.Equal(LdapAuthFailure.UserNotFound, first.Failure);
|
||||
|
||||
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
|
||||
Assert.Equal("dc2", factory.Created[2].ConnectArgs!.Value.Host);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AllEndpointsDown_LeavesPreferenceUnchanged()
|
||||
{
|
||||
// A total outage teaches nothing about which endpoint is healthy, so the walk
|
||||
// must not move the preference — the next login still starts at the primary.
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc1
|
||||
new FakeLdapConnection().ThrowOnConnect(), // login 1 @ dc2
|
||||
HealthyWithUser()); // login 2: must start at dc1 again
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory);
|
||||
|
||||
Assert.Equal(LdapAuthFailure.ServiceAccountBindFailed,
|
||||
(await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Failure);
|
||||
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
|
||||
|
||||
Assert.Equal("dc1", factory.Created[2].ConnectArgs!.Value.Host);
|
||||
}
|
||||
|
||||
// ---- back-compat ----
|
||||
|
||||
[Fact]
|
||||
public async Task NoFallbacks_SingleAttempt_UnchangedBehavior()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnConnect());
|
||||
var svc = new LdapAuthService(Opts(/* no fallbacks */), factory);
|
||||
|
||||
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.Equal(LdapAuthFailure.ServiceAccountBindFailed, result.Failure);
|
||||
Assert.Single(factory.Created);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Disabled_NeverOpensAConnection_EvenWithFallbacks()
|
||||
{
|
||||
var factory = new SequenceLdapConnectionFactory(HealthyWithUser());
|
||||
var svc = new LdapAuthService(Opts("dc2") with { Enabled = false }, factory);
|
||||
|
||||
var result = await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.Equal(LdapAuthFailure.Disabled, result.Failure);
|
||||
Assert.Empty(factory.Created);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MalformedFallbackEntry_IsSkippedDefensively()
|
||||
{
|
||||
// The validator rejects this at boot; a service constructed without validation
|
||||
// must degrade to the parseable subset, not throw.
|
||||
var factory = new SequenceLdapConnectionFactory(HealthyWithUser());
|
||||
var svc = new LdapAuthService(Opts(":::"), factory);
|
||||
|
||||
Assert.True((await svc.AuthenticateAsync("alice", "pw", CancellationToken.None)).Succeeded);
|
||||
Assert.Single(factory.Created);
|
||||
}
|
||||
|
||||
// ---- observability ----
|
||||
|
||||
/// <summary>Captures emitted log entries so failover visibility can be asserted.</summary>
|
||||
private sealed class RecordingLogger : Microsoft.Extensions.Logging.ILogger<LdapAuthService>
|
||||
{
|
||||
public List<(Microsoft.Extensions.Logging.LogLevel Level, string Message)> Entries { get; } = new();
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true;
|
||||
public void Log<TState>(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId,
|
||||
TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
=> Entries.Add((logLevel, formatter(state, exception)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failover_LogsWarningWithEndpointAndStage_AndInfoOnStickyChange()
|
||||
{
|
||||
var logger = new RecordingLogger();
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnConnect(), HealthyWithUser());
|
||||
var svc = new LdapAuthService(Opts("dc2:3894"), factory, logger);
|
||||
|
||||
await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.Contains(logger.Entries, e =>
|
||||
e.Level == Microsoft.Extensions.Logging.LogLevel.Warning
|
||||
&& e.Message.Contains("dc1:636") && e.Message.Contains("connect"));
|
||||
Assert.Contains(logger.Entries, e =>
|
||||
e.Level == Microsoft.Extensions.Logging.LogLevel.Information
|
||||
&& e.Message.Contains("dc2:3894"));
|
||||
// Never log secrets:
|
||||
Assert.DoesNotContain(logger.Entries, e => e.Message.Contains("svcpw") || e.Message.Contains("pw\""));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HappyPath_LogsNothing()
|
||||
{
|
||||
var logger = new RecordingLogger();
|
||||
var factory = new SequenceLdapConnectionFactory(HealthyWithUser());
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory, logger);
|
||||
|
||||
await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.Empty(logger.Entries);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ServiceBindFailure_LogsTheServiceBindStage_NotConnect()
|
||||
{
|
||||
var logger = new RecordingLogger();
|
||||
var factory = new SequenceLdapConnectionFactory(
|
||||
new FakeLdapConnection().ThrowOnServiceBind(), HealthyWithUser());
|
||||
var svc = new LdapAuthService(Opts("dc2"), factory, logger);
|
||||
|
||||
await svc.AuthenticateAsync("alice", "pw", CancellationToken.None);
|
||||
|
||||
Assert.Contains(logger.Entries, e =>
|
||||
e.Level == Microsoft.Extensions.Logging.LogLevel.Warning
|
||||
&& e.Message.Contains("dc1:636") && e.Message.Contains("service-bind"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
using ZB.MOM.WW.Auth.Ldap.Internal;
|
||||
|
||||
namespace ZB.MOM.WW.Auth.Ldap.Tests;
|
||||
|
||||
public class LdapEndpointParserTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("dc2.example.com", 636, "dc2.example.com", 636)] // bare host -> default port
|
||||
[InlineData("dc2.example.com:3894", 636, "dc2.example.com", 3894)] // explicit port wins
|
||||
[InlineData(" dc2.example.com : 3894 ", 636, "dc2.example.com", 3894)] // whitespace tolerated
|
||||
public void TryParse_ValidEntry_Succeeds(string entry, int defaultPort, string host, int port)
|
||||
{
|
||||
Assert.True(LdapEndpointParser.TryParse(entry, defaultPort, out var ep, out _));
|
||||
Assert.Equal(host, ep.Host);
|
||||
Assert.Equal(port, ep.Port);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")] // empty
|
||||
[InlineData(" ")] // whitespace
|
||||
[InlineData(":636")] // empty host
|
||||
[InlineData("dc2:0")] // port below range
|
||||
[InlineData("dc2:65536")] // port above range
|
||||
[InlineData("dc2:abc")] // non-numeric port
|
||||
[InlineData("2001:db8::1:636")] // IPv6 literal (unsupported, >1 colon)
|
||||
public void TryParse_InvalidEntry_Fails(string entry)
|
||||
{
|
||||
Assert.False(LdapEndpointParser.TryParse(entry, 636, out _, out var error));
|
||||
Assert.NotEmpty(error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Endpoint_ToString_IsHostColonPort() =>
|
||||
Assert.Equal("dc2:3894", new LdapEndpoint("dc2", 3894).ToString());
|
||||
}
|
||||
@@ -88,4 +88,32 @@ public class LdapOptionsValidatorTests
|
||||
ServiceAccountDn = "",
|
||||
})
|
||||
.Failed);
|
||||
|
||||
[Fact]
|
||||
public void Validator_Succeeds_WithValidFallbackServers() =>
|
||||
Assert.False(new LdapOptionsValidator()
|
||||
.Validate(null, Opts() with { FallbackServers = new[] { "dc2", "dc3:3894" } })
|
||||
.Failed);
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData(":636")]
|
||||
[InlineData("dc2:0")]
|
||||
[InlineData("dc2:notaport")]
|
||||
public void Validator_Fails_OnMalformedFallbackServer(string entry)
|
||||
{
|
||||
var result = new LdapOptionsValidator()
|
||||
.Validate(null, Opts() with { FallbackServers = new[] { entry } });
|
||||
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains(nameof(LdapOptions.FallbackServers), result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validator_Skips_FallbackServers_WhenDisabled() =>
|
||||
// Same inert-when-disabled rule as every other connection field: a disabled
|
||||
// provider never opens a connection, so its fallback list is never walked.
|
||||
Assert.False(new LdapOptionsValidator()
|
||||
.Validate(null, Opts() with { Enabled = false, FallbackServers = new[] { ":::" } })
|
||||
.Failed);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
using Novell.Directory.Ldap;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.Auth.Ldap.Internal;
|
||||
|
||||
namespace ZB.MOM.WW.Auth.Ldap.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Covers <see cref="NovellLdapConnection.ReadEntries"/>' handling of AD continuation
|
||||
/// referrals. Active Directory returns one for any search based at a domain root, and
|
||||
/// Novell surfaces it by throwing from <c>Next()</c>. Before this was handled, the
|
||||
/// exception escaped the search stage and every login against
|
||||
/// <c>DC=nam,DC=zimmer,DC=com</c> failed as <c>ServiceAccountBindFailed</c> while the
|
||||
/// directory and credentials were healthy.
|
||||
/// </summary>
|
||||
public sealed class NovellSearchReferralTests
|
||||
{
|
||||
[Fact]
|
||||
public void Referral_before_the_only_entry_does_not_lose_it()
|
||||
{
|
||||
var results = new ScriptedResults(
|
||||
Step.Referral(),
|
||||
Step.Entry("CN=dohertj2,OU=Office Users,OU=Warsaw,DC=nam,DC=zimmer,DC=com"));
|
||||
|
||||
var entries = NovellLdapConnection.ReadEntries(results);
|
||||
|
||||
var entry = Assert.Single(entries);
|
||||
Assert.Equal("CN=dohertj2,OU=Office Users,OU=Warsaw,DC=nam,DC=zimmer,DC=com", entry.Dn);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Referral_after_the_entry_does_not_discard_what_was_already_read()
|
||||
{
|
||||
var results = new ScriptedResults(
|
||||
Step.Entry("CN=a,DC=nam,DC=zimmer,DC=com"),
|
||||
Step.Referral());
|
||||
|
||||
var entries = NovellLdapConnection.ReadEntries(results);
|
||||
|
||||
Assert.Equal("CN=a,DC=nam,DC=zimmer,DC=com", Assert.Single(entries).Dn);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The real shape observed against the NAM domain root: several references interleaved
|
||||
/// with the one user entry. All three of AD's usual continuation points are present.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Multiple_interleaved_referrals_still_yield_every_entry()
|
||||
{
|
||||
var results = new ScriptedResults(
|
||||
Step.Referral(),
|
||||
Step.Entry("CN=a,DC=nam,DC=zimmer,DC=com"),
|
||||
Step.Referral(),
|
||||
Step.Referral(),
|
||||
Step.Entry("CN=b,DC=nam,DC=zimmer,DC=com"));
|
||||
|
||||
var entries = NovellLdapConnection.ReadEntries(results);
|
||||
|
||||
Assert.Equal(["CN=a,DC=nam,DC=zimmer,DC=com", "CN=b,DC=nam,DC=zimmer,DC=com"],
|
||||
entries.Select(e => e.Dn));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A referral-only result set is a legitimate "nothing here" answer, and must come back
|
||||
/// empty rather than throwing — <c>LdapAuthService</c> reads empty as <c>UserNotFound</c>,
|
||||
/// which is the honest verdict, where an exception would become a system-side failure.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Referrals_only_returns_empty_rather_than_throwing()
|
||||
{
|
||||
var results = new ScriptedResults(Step.Referral(), Step.Referral());
|
||||
|
||||
Assert.Empty(NovellLdapConnection.ReadEntries(results));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A base <see cref="LdapException"/> carrying the Referral result code is treated the
|
||||
/// same as the derived type — the catch filter must not depend on the exact class.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Referral_result_code_on_a_base_exception_is_also_skipped()
|
||||
{
|
||||
var results = new ScriptedResults(
|
||||
Step.Throwing(new LdapException("referral", LdapException.Referral, null)),
|
||||
Step.Entry("CN=a,DC=nam,DC=zimmer,DC=com"));
|
||||
|
||||
Assert.Single(NovellLdapConnection.ReadEntries(results));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Only referrals are swallowed. A genuine directory fault must still propagate so the
|
||||
/// caller maps it to a system-side failure — silently returning a short result set would
|
||||
/// turn "the directory is broken" into "no such user", which is the wrong answer to give
|
||||
/// about someone's credentials.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_non_referral_fault_still_propagates()
|
||||
{
|
||||
var results = new ScriptedResults(
|
||||
Step.Throwing(new LdapException("busy", LdapException.Busy, null)),
|
||||
Step.Entry("CN=a,DC=nam,DC=zimmer,DC=com"));
|
||||
|
||||
var ex = Assert.Throws<LdapException>(() => NovellLdapConnection.ReadEntries(results));
|
||||
Assert.Equal(LdapException.Busy, ex.ResultCode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A pathological client that throws a referral without advancing must not spin forever
|
||||
/// inside a fail-closed authentication path.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void An_endless_referral_stream_terminates()
|
||||
{
|
||||
var results = new EndlessReferrals();
|
||||
|
||||
var entries = NovellLdapConnection.ReadEntries(results);
|
||||
|
||||
Assert.Empty(entries);
|
||||
Assert.InRange(results.Calls, 1, 2_000);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
private sealed record Step(LdapEntry? Value, Exception? Throw)
|
||||
{
|
||||
public static Step Entry(string dn) => new(new LdapEntry(dn, new LdapAttributeSet()), null);
|
||||
public static Step Referral() => new(null, new LdapReferralException("continuation reference"));
|
||||
public static Step Throwing(Exception ex) => new(null, ex);
|
||||
}
|
||||
|
||||
/// <summary>Replays a fixed script of entries and faults through the Novell result seam.</summary>
|
||||
private sealed class ScriptedResults : ILdapSearchResults
|
||||
{
|
||||
private readonly Step[] _steps;
|
||||
private int _index;
|
||||
|
||||
public ScriptedResults(params Step[] steps) => _steps = steps;
|
||||
|
||||
public int Count => _steps.Length;
|
||||
|
||||
public LdapControl[] ResponseControls => [];
|
||||
|
||||
public bool HasMore() => _index < _steps.Length;
|
||||
|
||||
public LdapEntry Next()
|
||||
{
|
||||
var step = _steps[_index++];
|
||||
if (step.Throw is not null) throw step.Throw;
|
||||
return step.Value!;
|
||||
}
|
||||
|
||||
// The production reader drives HasMore()/Next() directly; enumeration is part of the
|
||||
// interface but not part of the path under test.
|
||||
public IEnumerator<LdapEntry> GetEnumerator() => throw new NotSupportedException();
|
||||
|
||||
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
|
||||
}
|
||||
|
||||
/// <summary>Always claims more results and always throws a referral, never advancing.</summary>
|
||||
private sealed class EndlessReferrals : ILdapSearchResults
|
||||
{
|
||||
public int Calls { get; private set; }
|
||||
|
||||
public int Count => int.MaxValue;
|
||||
|
||||
public LdapControl[] ResponseControls => [];
|
||||
|
||||
public bool HasMore() => true;
|
||||
|
||||
public LdapEntry Next()
|
||||
{
|
||||
Calls++;
|
||||
throw new LdapReferralException("continuation reference");
|
||||
}
|
||||
|
||||
public IEnumerator<LdapEntry> GetEnumerator() => throw new NotSupportedException();
|
||||
|
||||
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using ZB.MOM.WW.Auth.Ldap.Internal;
|
||||
|
||||
namespace ZB.MOM.WW.Auth.Ldap.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Test double for <see cref="ILdapConnectionFactory"/> that hands out pre-scripted
|
||||
/// <see cref="FakeLdapConnection"/> instances in order, one per <see cref="Create"/> call —
|
||||
/// exactly matching the failover walk, which opens a fresh connection per endpoint attempt.
|
||||
/// Records every created connection for inspection. When the script runs out, returns a
|
||||
/// bare unscripted fake (which yields user-not-found) so a miscounted test fails loudly
|
||||
/// rather than reusing a stale fake.
|
||||
/// </summary>
|
||||
internal sealed class SequenceLdapConnectionFactory : ILdapConnectionFactory
|
||||
{
|
||||
private readonly Queue<FakeLdapConnection> _script;
|
||||
|
||||
/// <summary>Scripts the connections handed out, in the order the walk will request them.</summary>
|
||||
public SequenceLdapConnectionFactory(params FakeLdapConnection[] fakes)
|
||||
=> _script = new Queue<FakeLdapConnection>(fakes);
|
||||
|
||||
/// <summary>Every connection handed out, in creation order (scripted or overflow).</summary>
|
||||
public List<FakeLdapConnection> Created { get; } = new();
|
||||
|
||||
/// <inheritdoc />
|
||||
public ILdapConnection Create()
|
||||
{
|
||||
var fake = _script.Count > 0 ? _script.Dequeue() : new FakeLdapConnection();
|
||||
Created.Add(fake);
|
||||
return fake;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Smoke test: verifies the sequencing and overflow behaviour the failover tests rely on.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
public class SequenceLdapConnectionFactorySmokeTests
|
||||
{
|
||||
[Fact]
|
||||
public void Create_ReturnsScriptedFakesInOrder_ThenOverflows()
|
||||
{
|
||||
var first = new FakeLdapConnection().ThrowOnConnect();
|
||||
var second = new FakeLdapConnection();
|
||||
var factory = new SequenceLdapConnectionFactory(first, second);
|
||||
|
||||
Assert.Same(first, factory.Create());
|
||||
Assert.Same(second, factory.Create());
|
||||
Assert.NotSame(second, factory.Create()); // overflow: fresh bare fake
|
||||
Assert.Equal(3, factory.Created.Count);
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ Health-check libraries for the **ZB.MOM.WW SCADA family** (OtOpcUa, MxAccessGate
|
||||
|
||||
The library normalizes the three-tier health endpoint convention (`/health/ready`, `/health/active`, `/healthz`) and provides reusable probe implementations so the three sister projects share a common surface without duplicating probe logic.
|
||||
|
||||
**Built at 0.1.0. NOT yet adopted by the three apps.** Adoption is tracked in `~/Desktop/scadaproj/components/health/GAPS.md`.
|
||||
**Built at 0.3.0 and published to the Gitea NuGet feed. Adopted by all four apps** — adoption is tracked in `~/Desktop/scadaproj/components/health/GAPS.md`.
|
||||
|
||||
---
|
||||
|
||||
@@ -13,7 +13,7 @@ The library normalizes the three-tier health endpoint convention (`/health/ready
|
||||
| Package | Responsibilities | Key Dependencies |
|
||||
|---|---|---|
|
||||
| `ZB.MOM.WW.Health` | Core tier convention, `MapZbHealth` extension, canonical JSON writer (`ZbHealthWriter`), `IActiveNodeGate` seam, `GrpcDependencyHealthCheck` reachability probe, tier-tag constants (`ZbHealthTags`). No Akka or EF dependency. | `Microsoft.AspNetCore.App` (framework ref), `Grpc.Net.Client` |
|
||||
| `ZB.MOM.WW.Health.Akka` | `AkkaClusterHealthCheck` with a configurable `AkkaClusterStatusPolicy` (presets: `Default` / `OtOpcUaCompat`), `ActiveNodeHealthCheck` with an optional role filter, and `AkkaActiveNodeGate` that backs `IActiveNodeGate` from cluster member state. | `ZB.MOM.WW.Health`, `Akka.Cluster` |
|
||||
| `ZB.MOM.WW.Health.Akka` | `AkkaClusterHealthCheck` with a configurable `AkkaClusterStatusPolicy` (presets: `Default` / `OtOpcUaCompat`); `ClusterActiveNode` — the family-wide **oldest-`Up`-member** rule, optionally role-scoped; `ActiveNodeHealthCheck` over it (role-preference list, `activeRole`/`activeNode` in the result `data`); and `AkkaActiveNodeGate`, which backs `IActiveNodeGate` from the same rule. | `ZB.MOM.WW.Health`, `Akka.Cluster` |
|
||||
| `ZB.MOM.WW.Health.EntityFrameworkCore` | `DatabaseHealthCheck<TContext>` — resolves `IDbContextFactory<TContext>` when registered, else a scoped `TContext`; pool-safe. Default probe: `CanConnectAsync`. Optional `ProbeQuery` delegate for query-based validation. | `ZB.MOM.WW.Health`, `Microsoft.EntityFrameworkCore` |
|
||||
|
||||
---
|
||||
@@ -46,14 +46,19 @@ dotnet test ZB.MOM.WW.Health.slnx
|
||||
dotnet pack ZB.MOM.WW.Health.slnx -c Release -o ./artifacts
|
||||
```
|
||||
|
||||
All three test assemblies run offline:
|
||||
All three test assemblies run offline. The Akka suite forms real in-process clusters on loopback: a
|
||||
single-node one for the cluster-data payload, and a **two-node one built so the oldest member is not
|
||||
the lowest-addressed one** — the only condition under which age- and leader-ordering disagree, and so
|
||||
the only fixture that can catch a leader-based active-node regression. It binds fixed ports
|
||||
19540/19541 (distinct from OtOpcUa's equivalent fixture) and carries a sanity check that the
|
||||
divergence actually occurred, so the assertions cannot pass for the wrong reason:
|
||||
|
||||
| Assembly | Tests |
|
||||
|---|---|
|
||||
| `ZB.MOM.WW.Health.Tests` | 20 |
|
||||
| `ZB.MOM.WW.Health.Akka.Tests` | 32 |
|
||||
| `ZB.MOM.WW.Health.Tests` | 25 |
|
||||
| `ZB.MOM.WW.Health.Akka.Tests` | 45 |
|
||||
| `ZB.MOM.WW.Health.EntityFrameworkCore.Tests` | 6 |
|
||||
| **Total** | **58** |
|
||||
| **Total** | **76** |
|
||||
|
||||
`GeneratePackageOnBuild` is off — pack explicitly with the command above.
|
||||
|
||||
@@ -61,7 +66,7 @@ All three test assemblies run offline:
|
||||
|
||||
## Status
|
||||
|
||||
Built at **0.1.0** and published to the Gitea NuGet feed. **Not yet adopted** by the three apps — adoption is tracked in the component backlog:
|
||||
Built at **0.3.0** and published to the Gitea NuGet feed. **Adopted** by all four apps — adoption is tracked in the component backlog:
|
||||
|
||||
- `~/Desktop/scadaproj/components/health/GAPS.md` — adoption order, effort, and risk
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Version>0.1.0</Version>
|
||||
<Version>0.3.0</Version>
|
||||
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
|
||||
<!-- Emit XML docs so the public API summaries ship inside the packed nupkgs (IntelliSense for
|
||||
consumers). CS1591 (missing doc on a public member) is suppressed so undocumented test /
|
||||
|
||||
@@ -30,7 +30,19 @@ MxAccessGateway consumes the core package only — it has no Akka cluster and no
|
||||
|
||||
## Versioning
|
||||
|
||||
All three packages are versioned **lockstep** from `Directory.Build.props`. The current release is **0.1.0**. A single version bump in `Directory.Build.props` bumps all three packages simultaneously — consumers should reference the same version for all ZB.MOM.WW.Health packages.
|
||||
All three packages are versioned **lockstep** from `Directory.Build.props`. The current release is **0.2.0**. A single version bump in `Directory.Build.props` bumps all three packages simultaneously — consumers should reference the same version for all ZB.MOM.WW.Health packages.
|
||||
|
||||
### 0.2.0 — per-entry `data` (additive, non-breaking)
|
||||
|
||||
- `ZbHealthWriter` emits an optional `"data": { … }` object per entry, sourced from the check's
|
||||
`HealthCheckResult.Data`, **only when non-empty** — a check that publishes no data produces a
|
||||
byte-identical body to 0.1.0, so every existing consumer is unaffected. Data keys are written
|
||||
verbatim (the camelCase policy applies to the envelope, not to dictionary keys).
|
||||
- `AkkaClusterHealthCheck` populates that dictionary with this node's cluster view: `leader`
|
||||
(omitted while unknown), `selfAddress`, `selfRoles`, `memberCount`, `unreachableCount`. The
|
||||
startup-safety paths (no `ActorSystem`, cluster not yet accessible) stay description-only.
|
||||
- Consumers gain the leader field on a package bump alone — no application code change, provided the
|
||||
app registers the shared `AkkaClusterHealthCheck`.
|
||||
|
||||
---
|
||||
|
||||
@@ -42,14 +54,16 @@ dotnet build ZB.MOM.WW.Health.slnx
|
||||
dotnet test ZB.MOM.WW.Health.slnx
|
||||
```
|
||||
|
||||
All three test assemblies run with `dotnet test` and require no external dependencies (no running Akka cluster, no database):
|
||||
All three test assemblies run with `dotnet test` and require no external dependencies — no database and no
|
||||
external cluster (the cluster-data tests form a single-node in-process Akka cluster on a loopback port
|
||||
via `Akka.TestKit.Xunit2`):
|
||||
|
||||
| Assembly | Tests |
|
||||
|---|---|
|
||||
| `ZB.MOM.WW.Health.Tests` | 20 |
|
||||
| `ZB.MOM.WW.Health.Akka.Tests` | 32 |
|
||||
| `ZB.MOM.WW.Health.Tests` | 25 |
|
||||
| `ZB.MOM.WW.Health.Akka.Tests` | 39 |
|
||||
| `ZB.MOM.WW.Health.EntityFrameworkCore.Tests` | 6 |
|
||||
| **Total** | **58** |
|
||||
| **Total** | **70** |
|
||||
|
||||
---
|
||||
|
||||
@@ -62,9 +76,9 @@ dotnet pack ZB.MOM.WW.Health.slnx -c Release -o ./artifacts
|
||||
Produces three `.nupkg` files in `artifacts/`:
|
||||
|
||||
```
|
||||
ZB.MOM.WW.Health.0.1.0.nupkg
|
||||
ZB.MOM.WW.Health.Akka.0.1.0.nupkg
|
||||
ZB.MOM.WW.Health.EntityFrameworkCore.0.1.0.nupkg
|
||||
ZB.MOM.WW.Health.0.2.0.nupkg
|
||||
ZB.MOM.WW.Health.Akka.0.2.0.nupkg
|
||||
ZB.MOM.WW.Health.EntityFrameworkCore.0.2.0.nupkg
|
||||
```
|
||||
|
||||
`GeneratePackageOnBuild` is off — pack explicitly as above.
|
||||
@@ -73,7 +87,7 @@ ZB.MOM.WW.Health.EntityFrameworkCore.0.1.0.nupkg
|
||||
|
||||
## Status
|
||||
|
||||
**Built at 0.1.0. NOT yet adopted by the three apps.** Adoption is tracked in the component backlog:
|
||||
**Built at 0.2.0 and published to the Gitea NuGet feed. Adopted by all four apps** (OtOpcUa, MxAccessGateway, ScadaBridge, HistorianGateway). Adoption is tracked in the component backlog:
|
||||
|
||||
- `~/Desktop/scadaproj/components/health/GAPS.md`
|
||||
|
||||
|
||||
@@ -9,89 +9,80 @@ using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
namespace ZB.MOM.WW.Health.Akka;
|
||||
|
||||
/// <summary>
|
||||
/// Pure decision function for the active / leader probe, factored out of
|
||||
/// <see cref="ActiveNodeHealthCheck"/> so the role-less and role-filtered matrices are exhaustively
|
||||
/// table-testable without forming a real cluster.
|
||||
/// </summary>
|
||||
internal static class ActiveNodeDecision
|
||||
{
|
||||
/// <summary>
|
||||
/// Maps the resolved cluster facts to a <see cref="HealthStatus"/>.
|
||||
/// </summary>
|
||||
/// <param name="selfUp">Whether the local node's member status is <c>Up</c>.</param>
|
||||
/// <param name="isLeader">
|
||||
/// Whether the local node is the leader: the cluster leader in role-less mode, or the
|
||||
/// role-singleton leader in role-filtered mode.
|
||||
/// </param>
|
||||
/// <param name="hasRole">
|
||||
/// Whether the local node carries <paramref name="requiredRole"/>. Ignored when
|
||||
/// <paramref name="requiredRole"/> is <c>null</c>.
|
||||
/// </param>
|
||||
/// <param name="requiredRole">
|
||||
/// The role to scope the check to, or <c>null</c> for the role-less (whole-cluster-leader) mode.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// Role-less: Healthy iff the node is Up and the cluster leader, otherwise Unhealthy.
|
||||
/// Role-filtered: Healthy when the node lacks the role (probe irrelevant) or carries the role and
|
||||
/// is the role-singleton leader; Degraded when it carries the role but is not the leader.
|
||||
/// </returns>
|
||||
public static HealthStatus Evaluate(bool selfUp, bool isLeader, bool hasRole, string? requiredRole)
|
||||
{
|
||||
if (requiredRole is null)
|
||||
return selfUp && isLeader ? HealthStatus.Healthy : HealthStatus.Unhealthy;
|
||||
|
||||
if (!hasRole)
|
||||
return HealthStatus.Healthy;
|
||||
|
||||
return isLeader ? HealthStatus.Healthy : HealthStatus.Degraded;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Health check that reports whether this node is the designated active / leader node.
|
||||
/// An optional role scopes the check to nodes carrying that role. Register to the
|
||||
/// <see cref="ZbHealthTags.Active"/> tag.
|
||||
/// Active-tier probe: Healthy (200) on the one node in charge, Unhealthy (503) on its standby.
|
||||
/// Register to the <see cref="ZbHealthTags.Active"/> tag.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The <see cref="ActorSystem"/> is resolved lazily from the service provider. If it is not yet
|
||||
/// available — e.g. during startup before Akka is initialised — the check returns
|
||||
/// <see cref="HealthStatus.Degraded"/> rather than throwing, so it is startup-safe.
|
||||
/// <para>
|
||||
/// "In charge" means the <b>oldest Up member</b> — see <see cref="ClusterActiveNode"/> for
|
||||
/// why leadership is the wrong answer and what breaks when it is used. Scope the competition
|
||||
/// to a role with <see cref="ActiveNodeHealthCheckOptions.RolePreference"/>; leave it empty
|
||||
/// for a cluster whose nodes all compete for the same single active slot.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Behaviour change in 0.3.0.</b> Through 0.2.1 this check selected by
|
||||
/// <c>ClusterState.Leader</c> / <c>RoleLeader</c>, and its role-filtered mode reported
|
||||
/// Healthy for any node <i>lacking</i> the role. Both were wrong, both were found on live
|
||||
/// rigs, and both Akka consumers had already replaced it with a private copy rather than use
|
||||
/// it. It now selects by age and reports <see cref="HealthStatus.Unhealthy"/> for a node that
|
||||
/// owns no active work. A node whose active tier previously answered 200 unconditionally will
|
||||
/// now answer 503 unless it really is the active node — which is the point.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Startup-safe: before the <see cref="ActorSystem"/> or the cluster is available the result
|
||||
/// is <see cref="HealthStatus.Degraded"/> (a 200 under the tier mapping), so a node that is
|
||||
/// merely still booting is not reported as a failed standby and pulled from rotation on the
|
||||
/// way up.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class ActiveNodeHealthCheck : IHealthCheck
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly string? _role;
|
||||
private readonly ActiveNodeHealthCheckOptions _options;
|
||||
|
||||
/// <summary>
|
||||
/// Role-less constructor: Healthy when the node is <c>Up</c> and the cluster leader
|
||||
/// (ScadaBridge ActiveNode pattern); Unhealthy otherwise. Degraded when the ActorSystem /
|
||||
/// cluster is not yet ready.
|
||||
/// Unscoped: the active node is the oldest Up member of the whole cluster.
|
||||
/// </summary>
|
||||
/// <param name="serviceProvider">
|
||||
/// The application service provider. The <see cref="ActorSystem"/> is resolved lazily so the
|
||||
/// check is startup-safe: if no <see cref="ActorSystem"/> is registered yet the result is Degraded.
|
||||
/// check is startup-safe.
|
||||
/// </param>
|
||||
public ActiveNodeHealthCheck(IServiceProvider serviceProvider)
|
||||
: this(serviceProvider, new ActiveNodeHealthCheckOptions())
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_role = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Role-filtered constructor: Healthy when the node lacks <paramref name="role"/> or carries it
|
||||
/// and is the role-singleton leader; Degraded when it carries the role but is not the leader
|
||||
/// (OtOpcUa AdminRoleLeader pattern). Degraded when the ActorSystem / cluster is not yet ready.
|
||||
/// Role-scoped: the active node is the oldest Up member carrying <paramref name="role"/>.
|
||||
/// </summary>
|
||||
/// <param name="serviceProvider">
|
||||
/// The application service provider. The <see cref="ActorSystem"/> is resolved lazily so the
|
||||
/// check is startup-safe: if no <see cref="ActorSystem"/> is registered yet the result is Degraded.
|
||||
/// check is startup-safe.
|
||||
/// </param>
|
||||
/// <param name="role">The Akka cluster role to scope the check to.</param>
|
||||
/// <param name="role">The cluster role to scope the competition to.</param>
|
||||
public ActiveNodeHealthCheck(IServiceProvider serviceProvider, string role)
|
||||
: this(serviceProvider, NewOptions(role))
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configured with an explicit <see cref="ActiveNodeHealthCheckOptions"/>.
|
||||
/// </summary>
|
||||
/// <param name="serviceProvider">
|
||||
/// The application service provider. The <see cref="ActorSystem"/> is resolved lazily so the
|
||||
/// check is startup-safe.
|
||||
/// </param>
|
||||
/// <param name="options">The role-scoping options.</param>
|
||||
public ActiveNodeHealthCheck(IServiceProvider serviceProvider, ActiveNodeHealthCheckOptions options)
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
private static ActiveNodeHealthCheckOptions NewOptions(string role)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(role);
|
||||
_role = role;
|
||||
return new ActiveNodeHealthCheckOptions { RolePreference = new[] { role } };
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -103,10 +94,8 @@ public sealed class ActiveNodeHealthCheck : IHealthCheck
|
||||
if (system is null)
|
||||
return Task.FromResult(HealthCheckResult.Degraded("ActorSystem not yet available."));
|
||||
|
||||
MemberStatus selfStatus;
|
||||
bool selfUp;
|
||||
bool hasRole;
|
||||
bool isLeader;
|
||||
Member self;
|
||||
IEnumerable<Member> members;
|
||||
try
|
||||
{
|
||||
// Reading cluster membership can throw while the ActorSystem exists but the cluster has
|
||||
@@ -114,54 +103,49 @@ public sealed class ActiveNodeHealthCheck : IHealthCheck
|
||||
// ConfigurationException). The spec's startup-safety rule maps this to Degraded rather
|
||||
// than letting the exception escape (which the host would record as Unhealthy).
|
||||
var cluster = Cluster.Get(system);
|
||||
var self = cluster.SelfMember;
|
||||
selfStatus = self.Status;
|
||||
selfUp = selfStatus == MemberStatus.Up;
|
||||
|
||||
if (_role is null)
|
||||
{
|
||||
hasRole = false;
|
||||
var leader = cluster.State.Leader;
|
||||
isLeader = leader is not null && leader == self.Address;
|
||||
}
|
||||
else
|
||||
{
|
||||
hasRole = self.HasRole(_role);
|
||||
var roleLeader = cluster.State.RoleLeader(_role);
|
||||
isLeader = roleLeader is not null && roleLeader == self.Address;
|
||||
}
|
||||
self = cluster.SelfMember;
|
||||
members = cluster.State.Members;
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
return Task.FromResult(HealthCheckResult.Degraded("Akka cluster state not yet accessible.", ex));
|
||||
}
|
||||
|
||||
var health = ActiveNodeDecision.Evaluate(selfUp, isLeader, hasRole, _role);
|
||||
var description = DescribeResult(health, selfStatus, selfUp, isLeader);
|
||||
var result = health switch
|
||||
{
|
||||
HealthStatus.Healthy => HealthCheckResult.Healthy(description),
|
||||
HealthStatus.Degraded => HealthCheckResult.Degraded(description),
|
||||
_ => HealthCheckResult.Unhealthy(description),
|
||||
};
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
var scoped = _options.RolePreference.Count > 0;
|
||||
var role = ClusterActiveNode.ResolveActiveRole(self.Roles, _options.RolePreference);
|
||||
|
||||
private string DescribeResult(HealthStatus health, MemberStatus status, bool selfUp, bool isLeader)
|
||||
{
|
||||
if (_role is null)
|
||||
var data = new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
if (health == HealthStatus.Healthy)
|
||||
return "Active node (cluster leader).";
|
||||
return selfUp && !isLeader
|
||||
? "Standby: node is Up but not the cluster leader."
|
||||
: $"Standby: node is not Up (status: {status}).";
|
||||
["selfAddress"] = self.Address.ToString(),
|
||||
};
|
||||
if (role is not null) data["activeRole"] = role;
|
||||
|
||||
if (scoped && role is null)
|
||||
{
|
||||
// The node carries none of the preferred roles, so it owns no active work and cannot be
|
||||
// the active node for any of them. Reported per NoActiveRoleStatus — Unhealthy by
|
||||
// default, because the active tier's whole job is to keep such a node out of the pool.
|
||||
var message = $"Node carries none of the active roles: {string.Join(", ", _options.RolePreference)}.";
|
||||
return Task.FromResult(new HealthCheckResult(_options.NoActiveRoleStatus, message, exception: null, data));
|
||||
}
|
||||
|
||||
return health switch
|
||||
var active = ClusterActiveNode.OldestUpMember(members, role);
|
||||
if (active is not null) data["activeNode"] = active.Address.ToString();
|
||||
|
||||
var scopeLabel = role is null ? "the cluster" : $"role '{role}'";
|
||||
|
||||
if (active is null)
|
||||
{
|
||||
HealthStatus.Healthy => $"Active for role '{_role}' (or not a role member).",
|
||||
_ => $"Role '{_role}' member but not leader.",
|
||||
};
|
||||
// Nobody is Up in the scope — a cold start, or every holder is still Joining or already
|
||||
// Leaving. Nobody is active, so nobody may claim to be.
|
||||
return Task.FromResult(HealthCheckResult.Unhealthy(
|
||||
$"No Up member currently holds the active slot for {scopeLabel}.", exception: null, data));
|
||||
}
|
||||
|
||||
// Compared by UniqueAddress: a node that restarted on the same host and port is a different
|
||||
// member, and during the overlap the cluster briefly holds both incarnations.
|
||||
return Task.FromResult(active.UniqueAddress.Equals(self.UniqueAddress)
|
||||
? HealthCheckResult.Healthy($"Active node for {scopeLabel} (oldest Up member).", data)
|
||||
: HealthCheckResult.Unhealthy($"Standby: active node for {scopeLabel} is {active.Address}.", exception: null, data));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
|
||||
namespace ZB.MOM.WW.Health.Akka;
|
||||
|
||||
/// <summary>
|
||||
/// Configures how <see cref="ActiveNodeHealthCheck"/> decides which role's active node this node is
|
||||
/// competing to be.
|
||||
/// </summary>
|
||||
public sealed class ActiveNodeHealthCheckOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Candidate roles in descending precedence. The check answers for the first one the local node
|
||||
/// carries. Empty (the default) means unscoped: the active node is the oldest Up member of the
|
||||
/// whole cluster.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A fused node should list the role its singletons are pinned to first — e.g.
|
||||
/// <c>["admin", "driver"]</c> for a node that hosts the admin singletons when fused and competes
|
||||
/// as a plain driver when it is a driver-only node.
|
||||
/// </remarks>
|
||||
public IReadOnlyList<string> RolePreference { get; init; } = Array.Empty<string>();
|
||||
|
||||
/// <summary>
|
||||
/// The status to report when <see cref="RolePreference"/> is non-empty and the node carries none
|
||||
/// of those roles — it owns no active work at all. Defaults to
|
||||
/// <see cref="HealthStatus.Unhealthy"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Unhealthy is the safe default and is deliberately <b>not</b> the "not applicable ⇒ Healthy"
|
||||
/// convention used by readiness probes. The active tier exists so a load balancer can pick
|
||||
/// exactly one node; a node that owns no active work must not be in that pool. Answering Healthy
|
||||
/// there is what made the previous role-filtered check report 200 on every node and silently
|
||||
/// broke leader-pinning (lmxopcua#494). Override only when a node genuinely should stay in the
|
||||
/// active pool without holding any of the preferred roles.
|
||||
/// </remarks>
|
||||
public HealthStatus NoActiveRoleStatus { get; init; } = HealthStatus.Unhealthy;
|
||||
}
|
||||
@@ -6,27 +6,47 @@ namespace ZB.MOM.WW.Health.Akka;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IActiveNodeGate"/> implementation that computes <see cref="IsActiveNode"/> directly
|
||||
/// from the Akka cluster state (self member <c>Up</c> and the local node is the cluster leader).
|
||||
/// Register as a singleton.
|
||||
/// from the Akka cluster state — the local node is the <b>oldest Up member</b>, optionally within a
|
||||
/// role scope. Register as a singleton.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The <see cref="ActorSystem"/> is resolved lazily from the service provider; if it is not yet
|
||||
/// available — e.g. during startup before Akka is initialised — <see cref="IsActiveNode"/> returns
|
||||
/// <c>false</c> (the safe default during startup). This gate reads the cluster state directly and
|
||||
/// does not resolve <see cref="ActiveNodeHealthCheck"/> from DI.
|
||||
/// <para>
|
||||
/// Shares its rule with <see cref="ActiveNodeHealthCheck"/> via
|
||||
/// <see cref="ClusterActiveNode"/>, so an endpoint gated by this type and the
|
||||
/// <c>/health/active</c> probe an orchestrator routes by can never disagree about which node
|
||||
/// is in charge.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Behaviour change in 0.3.0:</b> through 0.2.1 this gate selected by
|
||||
/// <c>ClusterState.Leader</c>, which is address-ordered and diverges from singleton placement
|
||||
/// after any restart. See <see cref="ClusterActiveNode"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The <see cref="ActorSystem"/> is resolved lazily from the service provider; if it is not
|
||||
/// yet available — e.g. during startup before Akka is initialised —
|
||||
/// <see cref="IsActiveNode"/> returns <c>false</c> (the safe default during startup, matching
|
||||
/// the standby case). This gate reads the cluster state directly and does not resolve
|
||||
/// <see cref="ActiveNodeHealthCheck"/> from DI.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class AkkaActiveNodeGate : IActiveNodeGate
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly string? _role;
|
||||
|
||||
/// <summary>Initializes a new <see cref="AkkaActiveNodeGate"/>.</summary>
|
||||
/// <param name="serviceProvider">
|
||||
/// The application service provider. The <see cref="ActorSystem"/> is resolved lazily; if it is
|
||||
/// not yet available <see cref="IsActiveNode"/> returns <c>false</c>.
|
||||
/// </param>
|
||||
public AkkaActiveNodeGate(IServiceProvider serviceProvider)
|
||||
/// <param name="role">
|
||||
/// The role to scope the competition to, or <c>null</c> (the default) to consider every Up
|
||||
/// member.
|
||||
/// </param>
|
||||
public AkkaActiveNodeGate(IServiceProvider serviceProvider, string? role = null)
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_role = role;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -38,13 +58,16 @@ public sealed class AkkaActiveNodeGate : IActiveNodeGate
|
||||
if (system is null)
|
||||
return false;
|
||||
|
||||
var cluster = Cluster.Get(system);
|
||||
var self = cluster.SelfMember;
|
||||
if (self.Status != MemberStatus.Up)
|
||||
try
|
||||
{
|
||||
return ClusterActiveNode.SelfIsActive(Cluster.Get(system), _role);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// ActorSystem exists but the cluster is not initialised yet. Same startup-safety
|
||||
// rule as the health check, expressed as the gate's safe default: not active.
|
||||
return false;
|
||||
|
||||
var leader = cluster.State.Leader;
|
||||
return leader is not null && leader == self.Address;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ public sealed class AkkaClusterHealthCheck : IHealthCheck
|
||||
return Task.FromResult(HealthCheckResult.Degraded("ActorSystem not yet available."));
|
||||
|
||||
MemberStatus status;
|
||||
IReadOnlyDictionary<string, object> data;
|
||||
try
|
||||
{
|
||||
// Cluster.Get(system).SelfMember can throw while the ActorSystem exists but the cluster
|
||||
@@ -51,7 +52,9 @@ public sealed class AkkaClusterHealthCheck : IHealthCheck
|
||||
// ConfigurationException). The spec's startup-safety rule maps this to Degraded, not an
|
||||
// escaping exception (which the host would record as Unhealthy and pull the node from
|
||||
// rotation).
|
||||
status = Cluster.Get(system).SelfMember.Status;
|
||||
var cluster = Cluster.Get(system);
|
||||
status = cluster.SelfMember.Status;
|
||||
data = BuildClusterData(cluster);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
@@ -62,10 +65,47 @@ public sealed class AkkaClusterHealthCheck : IHealthCheck
|
||||
var description = $"Akka cluster member status: {status}";
|
||||
var result = health switch
|
||||
{
|
||||
HealthStatus.Healthy => HealthCheckResult.Healthy(description),
|
||||
HealthStatus.Degraded => HealthCheckResult.Degraded(description),
|
||||
_ => HealthCheckResult.Unhealthy(description),
|
||||
HealthStatus.Healthy => HealthCheckResult.Healthy(description, data),
|
||||
HealthStatus.Degraded => HealthCheckResult.Degraded(description, data: data),
|
||||
_ => HealthCheckResult.Unhealthy(description, data: data),
|
||||
};
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Projects this node's view of the cluster into the check's <c>data</c> dictionary.
|
||||
/// </summary>
|
||||
/// <param name="cluster">The cluster extension for the local <see cref="ActorSystem"/>.</param>
|
||||
/// <returns>
|
||||
/// JSON-friendly values only (string / int / string[]), so the canonical health writer can emit
|
||||
/// them verbatim: <c>leader</c> (omitted while the leader is unknown, e.g. pre-join or during
|
||||
/// convergence), <c>selfAddress</c>, <c>selfRoles</c> (sorted for a stable payload),
|
||||
/// <c>memberCount</c>, <c>unreachableCount</c>.
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// This is deliberately each node's <em>own</em> view rather than an authoritative fleet answer —
|
||||
/// a consumer that reads it from several members of the same cluster can compare them, and a
|
||||
/// disagreement is a split-brain tell.
|
||||
/// </remarks>
|
||||
internal static IReadOnlyDictionary<string, object> BuildClusterData(Cluster cluster)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(cluster);
|
||||
|
||||
var state = cluster.State;
|
||||
var data = new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
["selfAddress"] = cluster.SelfAddress.ToString(),
|
||||
["selfRoles"] = cluster.SelfMember.Roles.OrderBy(static r => r, StringComparer.Ordinal).ToArray(),
|
||||
["memberCount"] = state.Members.Count,
|
||||
["unreachableCount"] = state.Unreachable.Count,
|
||||
};
|
||||
|
||||
// Omitted rather than emitted as null: a missing key reads as "this node does not know yet",
|
||||
// which is exactly the pre-join / mid-convergence state.
|
||||
var leader = state.Leader;
|
||||
if (leader is not null)
|
||||
data["leader"] = leader.ToString();
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
using Akka.Cluster;
|
||||
|
||||
namespace ZB.MOM.WW.Health.Akka;
|
||||
|
||||
/// <summary>
|
||||
/// THE family-wide definition of "the active node": the <b>oldest Up member</b> of the cluster,
|
||||
/// optionally scoped to a role — the member <c>ClusterSingletonManager</c> places singletons on.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Cluster <b>leadership</b> (<c>ClusterState.Leader</c> / <c>RoleLeader</c>) is the wrong
|
||||
/// answer and must never be used for a product-level active/standby decision. Leadership is
|
||||
/// <i>address</i>-ordered (host, then port) and has no relationship to time; singleton
|
||||
/// placement is <i>age</i>-ordered. On a freshly-formed cluster the two agree, which is why
|
||||
/// the discrepancy hides in unit tests and in a rig that has never been restarted. They
|
||||
/// diverge permanently after any restart: the restarted node rejoins as the youngest but
|
||||
/// keeps its address, so if it holds the lower address it becomes leader while the
|
||||
/// singletons — and all the work they own — stay on the other node.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Both Akka consumers in the family independently hit this and each wrote its own
|
||||
/// replacement rather than use the shared leader-based check
|
||||
/// (ScadaBridge <c>OldestNodeActiveHealthCheck</c>, OtOpcUa <c>ClusterPrimaryHealthCheck</c>).
|
||||
/// This type is those two implementations promoted to one, so the health tier, the inbound
|
||||
/// API gate, the store-and-forward delivery gate and OtOpcUa's OPC UA <c>ServiceLevel</c>
|
||||
/// cannot drift apart about which node is in charge.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Only <see cref="MemberStatus.Up"/> members are eligible, matching singleton placement: a
|
||||
/// <c>Leaving</c> node is handing its singletons over and must not be named active.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class ClusterActiveNode
|
||||
{
|
||||
/// <summary>
|
||||
/// The oldest <see cref="MemberStatus.Up"/> member, optionally scoped to <paramref name="role"/>.
|
||||
/// </summary>
|
||||
/// <param name="members">The cluster members, in any order.</param>
|
||||
/// <param name="role">
|
||||
/// The role to scope the selection to, or <c>null</c> to consider every Up member.
|
||||
/// </param>
|
||||
/// <returns>The oldest eligible member, or <c>null</c> when none is Up.</returns>
|
||||
public static Member? OldestUpMember(IEnumerable<Member> members, string? role = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(members);
|
||||
|
||||
Member? oldest = null;
|
||||
foreach (var member in members)
|
||||
{
|
||||
if (member.Status != MemberStatus.Up) continue;
|
||||
if (role is not null && !member.HasRole(role)) continue;
|
||||
if (oldest is null || member.IsOlderThan(oldest)) oldest = member;
|
||||
}
|
||||
|
||||
return oldest;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The oldest <see cref="MemberStatus.Up"/> member of <paramref name="cluster"/>, optionally
|
||||
/// scoped to <paramref name="role"/>.
|
||||
/// </summary>
|
||||
/// <param name="cluster">The Akka cluster to evaluate.</param>
|
||||
/// <param name="role">
|
||||
/// The role to scope the selection to, or <c>null</c> to consider every Up member.
|
||||
/// </param>
|
||||
/// <returns>The oldest eligible member, or <c>null</c> when none is Up.</returns>
|
||||
public static Member? OldestUpMember(Cluster cluster, string? role = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(cluster);
|
||||
return OldestUpMember(cluster.State.Members, role);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the local node is the active node — Up, and the oldest Up member in the role scope.
|
||||
/// </summary>
|
||||
/// <param name="cluster">The Akka cluster to evaluate.</param>
|
||||
/// <param name="role">
|
||||
/// The role to scope the selection to, or <c>null</c> to consider every Up member.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// <c>true</c> when the local member is Up, carries <paramref name="role"/> when one is given,
|
||||
/// and no other eligible member is older.
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// Identity is compared by <see cref="Member.UniqueAddress"/>, not <c>Address</c>: a node that
|
||||
/// restarted on the same host and port is a different member, and during the overlap the cluster
|
||||
/// briefly holds both incarnations.
|
||||
/// </remarks>
|
||||
public static bool SelfIsActive(Cluster cluster, string? role = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(cluster);
|
||||
|
||||
var self = cluster.SelfMember;
|
||||
if (self.Status != MemberStatus.Up) return false;
|
||||
if (role is not null && !self.HasRole(role)) return false;
|
||||
|
||||
var oldest = OldestUpMember(cluster.State.Members, role);
|
||||
return oldest is not null && oldest.UniqueAddress.Equals(self.UniqueAddress);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The first role in <paramref name="rolePreference"/> that <paramref name="selfRoles"/> carries
|
||||
/// — the role whose oldest Up member owns this node's active work.
|
||||
/// </summary>
|
||||
/// <param name="selfRoles">The local member's cluster roles.</param>
|
||||
/// <param name="rolePreference">
|
||||
/// Candidate roles in descending precedence, e.g. <c>["admin", "driver"]</c> for a node that
|
||||
/// answers for <c>admin</c> when it is fused and for <c>driver</c> when it is a driver-only node.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// The first carried role, or <c>null</c> when the node carries none of them.
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// Precedence matters on a fused node: it must answer for the role its singletons — and the
|
||||
/// traffic being routed to it — are pinned to. Answering for a lower-precedence role could name a
|
||||
/// different node the moment a single-role member joins the mesh.
|
||||
/// </remarks>
|
||||
public static string? ResolveActiveRole(IEnumerable<string> selfRoles, IReadOnlyList<string> rolePreference)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(selfRoles);
|
||||
ArgumentNullException.ThrowIfNull(rolePreference);
|
||||
|
||||
if (rolePreference.Count == 0) return null;
|
||||
|
||||
var roles = selfRoles as IReadOnlyCollection<string> ?? selfRoles.ToList();
|
||||
foreach (var candidate in rolePreference)
|
||||
{
|
||||
if (roles.Contains(candidate)) return candidate;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
|
||||
@@ -16,15 +17,18 @@ namespace ZB.MOM.WW.Health;
|
||||
/// "status": "Healthy|Degraded|Unhealthy",
|
||||
/// "totalDurationMs": 12.34,
|
||||
/// "entries": {
|
||||
/// "<name>": { "status": "...", "description": "...", "durationMs": 1.23 }
|
||||
/// "<name>": { "status": "...", "description": "...", "durationMs": 1.23, "data": { ... } }
|
||||
/// }
|
||||
/// }
|
||||
/// </code>
|
||||
/// The <c>description</c> key is always present; when a check supplies no description it is emitted
|
||||
/// as JSON <c>null</c> (not omitted), matching the spec example and the <c>HealthChecks.UI.Client</c>
|
||||
/// shape. The HTTP status code is left to the ASP.NET Core health-checks middleware (Healthy/Degraded
|
||||
/// → 200, Unhealthy → 503); this writer only renders the body and sets
|
||||
/// <c>Content-Type: application/json</c>.
|
||||
/// shape. The <c>data</c> key is the opposite: it is emitted only when the check supplied a non-empty
|
||||
/// <see cref="HealthCheckResult.Data"/> dictionary, so payloads from checks that publish no data stay
|
||||
/// byte-identical to the pre-0.2.0 shape. Its keys are written verbatim (the camelCase naming policy
|
||||
/// applies to the envelope properties, not to dictionary keys). The HTTP status code is left to the
|
||||
/// ASP.NET Core health-checks middleware (Healthy/Degraded → 200, Unhealthy → 503); this writer only
|
||||
/// renders the body and sets <c>Content-Type: application/json</c>.
|
||||
/// </remarks>
|
||||
public static class ZbHealthWriter
|
||||
{
|
||||
@@ -60,6 +64,10 @@ public static class ZbHealthWriter
|
||||
Status = e.Value.Status.ToString(),
|
||||
Description = e.Value.Description,
|
||||
DurationMs = e.Value.Duration.TotalMilliseconds,
|
||||
// HealthReportEntry.Data is never null (empty when the check set none); mapping
|
||||
// empty → null is what drops the key entirely, keeping data-less payloads
|
||||
// byte-identical to the pre-0.2.0 shape.
|
||||
Data = e.Value.Data.Count > 0 ? e.Value.Data : null,
|
||||
}),
|
||||
};
|
||||
|
||||
@@ -78,5 +86,11 @@ public static class ZbHealthWriter
|
||||
public string Status { get; init; } = string.Empty;
|
||||
public string? Description { get; init; }
|
||||
public double DurationMs { get; init; }
|
||||
|
||||
// Per-property ignore, deliberately NOT a global DefaultIgnoreCondition: the writer must keep
|
||||
// emitting "description": null (see SerializerOptions above) while omitting "data" entirely
|
||||
// when a check publishes none.
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public IReadOnlyDictionary<string, object>? Data { get; init; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using ZB.MOM.WW.Health.Akka;
|
||||
|
||||
namespace ZB.MOM.WW.Health.Akka.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Table-driven tests for the pure <see cref="ActiveNodeDecision.Evaluate"/> helper covering both
|
||||
/// the role-less (ScadaBridge ActiveNode) and role-filtered (OtOpcUa AdminRoleLeader) matrices,
|
||||
/// plus the startup-safety null-guards on <see cref="ActiveNodeHealthCheck"/> and
|
||||
/// <see cref="AkkaActiveNodeGate"/> when no <see cref="ActorSystem"/> is registered.
|
||||
/// </summary>
|
||||
public sealed class ActiveNodeDecisionTests
|
||||
{
|
||||
// Role-less: requiredRole == null. hasRole is irrelevant. Healthy iff (selfUp && isLeader), else Unhealthy.
|
||||
public static IEnumerable<object?[]> RoleLessCases() => new[]
|
||||
{
|
||||
new object?[] { true, true, false, (string?)null, HealthStatus.Healthy },
|
||||
new object?[] { true, false, false, (string?)null, HealthStatus.Unhealthy },
|
||||
new object?[] { false, true, false, (string?)null, HealthStatus.Unhealthy },
|
||||
new object?[] { false, false, false, (string?)null, HealthStatus.Unhealthy },
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(RoleLessCases))]
|
||||
public void Evaluate_RoleLess(bool selfUp, bool isLeader, bool hasRole, string? requiredRole, HealthStatus expected)
|
||||
{
|
||||
Assert.Equal(expected, ActiveNodeDecision.Evaluate(selfUp, isLeader, hasRole, requiredRole));
|
||||
}
|
||||
|
||||
// Role-filtered: requiredRole != null.
|
||||
// lacks role -> Healthy (probe irrelevant for this node)
|
||||
// has role & is leader -> Healthy (selfUp is ignored — role-filtered mode only cares about leadership)
|
||||
// has role & not leader -> Degraded
|
||||
public static IEnumerable<object[]> RoleFilteredCases() => new[]
|
||||
{
|
||||
// node lacks the role -> Healthy regardless of selfUp / isLeader
|
||||
new object[] { true, true, false, "admin", HealthStatus.Healthy },
|
||||
new object[] { true, false, false, "admin", HealthStatus.Healthy },
|
||||
new object[] { false, false, false, "admin", HealthStatus.Healthy },
|
||||
// node carries the role and is leader -> Healthy (selfUp=true)
|
||||
new object[] { true, true, true, "admin", HealthStatus.Healthy },
|
||||
// node carries the role and is leader -> Healthy (selfUp=false: role-filtered mode ignores selfUp)
|
||||
new object[] { false, true, true, "admin", HealthStatus.Healthy },
|
||||
// node carries the role but is not leader -> Degraded
|
||||
new object[] { true, false, true, "admin", HealthStatus.Degraded },
|
||||
new object[] { false, false, true, "admin", HealthStatus.Degraded },
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(RoleFilteredCases))]
|
||||
public void Evaluate_RoleFiltered(bool selfUp, bool isLeader, bool hasRole, string? requiredRole, HealthStatus expected)
|
||||
{
|
||||
Assert.Equal(expected, ActiveNodeDecision.Evaluate(selfUp, isLeader, hasRole, requiredRole));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthCheck_RoleLess_NoActorSystem_ReturnsDegraded()
|
||||
{
|
||||
var provider = new ServiceCollection().BuildServiceProvider();
|
||||
var check = new ActiveNodeHealthCheck(provider);
|
||||
|
||||
var result = await check.CheckHealthAsync(NewContext(check));
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded, result.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthCheck_RoleFiltered_NoActorSystem_ReturnsDegraded()
|
||||
{
|
||||
var provider = new ServiceCollection().BuildServiceProvider();
|
||||
var check = new ActiveNodeHealthCheck(provider, "admin");
|
||||
|
||||
var result = await check.CheckHealthAsync(NewContext(check));
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded, result.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_NoActorSystem_IsActiveNodeFalse()
|
||||
{
|
||||
var provider = new ServiceCollection().BuildServiceProvider();
|
||||
var gate = new AkkaActiveNodeGate(provider);
|
||||
|
||||
Assert.False(gate.IsActiveNode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthCheck_RoleLess_ClusterInaccessible_ReturnsDegraded()
|
||||
{
|
||||
// ActorSystem present but Akka.Cluster not configured → Cluster.Get throws. The check must
|
||||
// return Degraded (startup-safety rule), not let the exception escape (→ Unhealthy).
|
||||
using var system = ActorSystem.Create("plain-no-cluster-roleless");
|
||||
try
|
||||
{
|
||||
var provider = new ServiceCollection()
|
||||
.AddSingleton(system)
|
||||
.BuildServiceProvider();
|
||||
var check = new ActiveNodeHealthCheck(provider);
|
||||
|
||||
var result = await check.CheckHealthAsync(NewContext(check));
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded, result.Status);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthCheck_RoleFiltered_ClusterInaccessible_ReturnsDegraded()
|
||||
{
|
||||
using var system = ActorSystem.Create("plain-no-cluster-rolefiltered");
|
||||
try
|
||||
{
|
||||
var provider = new ServiceCollection()
|
||||
.AddSingleton(system)
|
||||
.BuildServiceProvider();
|
||||
var check = new ActiveNodeHealthCheck(provider, "admin");
|
||||
|
||||
var result = await check.CheckHealthAsync(NewContext(check));
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded, result.Status);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
private static HealthCheckContext NewContext(IHealthCheck check) => new()
|
||||
{
|
||||
Registration = new HealthCheckRegistration("active-node", check, HealthStatus.Unhealthy, tags: null),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using ZB.MOM.WW.Health.Akka;
|
||||
|
||||
namespace ZB.MOM.WW.Health.Akka.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The states a node passes through on the way up, where the active tier must not report a booting
|
||||
/// node as a failed standby — now that the tier is a real 503, an Unhealthy here would pull the node
|
||||
/// out of an orchestrator's pool while it starts.
|
||||
/// </summary>
|
||||
public sealed class ActiveNodeStartupSafetyTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task NoActorSystem_IsDegraded()
|
||||
{
|
||||
var check = new ActiveNodeHealthCheck(new ServiceCollection().BuildServiceProvider());
|
||||
|
||||
var result = await check.CheckHealthAsync(NewContext(check));
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded, result.Status);
|
||||
// Description-only: an empty data dictionary is what makes the writer omit "data" entirely.
|
||||
Assert.Empty(result.Data);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NoActorSystem_RoleScoped_IsDegraded()
|
||||
{
|
||||
var check = new ActiveNodeHealthCheck(new ServiceCollection().BuildServiceProvider(), "admin");
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded, (await check.CheckHealthAsync(NewContext(check))).Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_NoActorSystem_IsNotActive()
|
||||
{
|
||||
Assert.False(new AkkaActiveNodeGate(new ServiceCollection().BuildServiceProvider()).IsActiveNode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ClusterNotConfigured_IsDegraded()
|
||||
{
|
||||
// ActorSystem present but Akka.Cluster not configured → Cluster.Get throws. The check must
|
||||
// return Degraded (the spec's startup-safety rule), not let the exception escape (→ Unhealthy).
|
||||
using var system = ActorSystem.Create("active-node-no-cluster");
|
||||
try
|
||||
{
|
||||
var provider = new ServiceCollection().AddSingleton(system).BuildServiceProvider();
|
||||
var check = new ActiveNodeHealthCheck(provider);
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded, (await check.CheckHealthAsync(NewContext(check))).Status);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Gate_ClusterNotConfigured_IsNotActive()
|
||||
{
|
||||
using var system = ActorSystem.Create("active-node-gate-no-cluster");
|
||||
try
|
||||
{
|
||||
var provider = new ServiceCollection().AddSingleton(system).BuildServiceProvider();
|
||||
|
||||
Assert.False(new AkkaActiveNodeGate(provider).IsActiveNode);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ClusteredButNeverJoined_IsUnhealthy_NobodyIsActive()
|
||||
{
|
||||
// A cold start: the cluster provider is up but no member has reached Up. Nobody is active, so
|
||||
// nobody may claim to be — this is the one case that must NOT be Degraded, because a cluster
|
||||
// that never forms would otherwise sit at 200 forever.
|
||||
var config = ConfigurationFactory.ParseString("""
|
||||
akka {
|
||||
loglevel = "WARNING"
|
||||
actor.provider = cluster
|
||||
remote.dot-netty.tcp { hostname = "127.0.0.1", port = 0 }
|
||||
cluster.seed-nodes = []
|
||||
}
|
||||
""");
|
||||
var system = ActorSystem.Create("active-node-never-joined", config);
|
||||
try
|
||||
{
|
||||
var provider = new ServiceCollection().AddSingleton(system).BuildServiceProvider();
|
||||
var check = new ActiveNodeHealthCheck(provider);
|
||||
|
||||
var result = await check.CheckHealthAsync(NewContext(check));
|
||||
|
||||
Assert.Equal(HealthStatus.Unhealthy, result.Status);
|
||||
Assert.Contains("No Up member", result.Description);
|
||||
Assert.False(new AkkaActiveNodeGate(provider).IsActiveNode);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
private static HealthCheckContext NewContext(IHealthCheck check) => new()
|
||||
{
|
||||
Registration = new HealthCheckRegistration("active-node", check, HealthStatus.Unhealthy, tags: null),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
using ZB.MOM.WW.Health.Akka;
|
||||
|
||||
namespace ZB.MOM.WW.Health.Akka.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Covers <see cref="ClusterActiveNode.ResolveActiveRole"/> — which role a node competes for. Pure,
|
||||
/// so no cluster is needed; the age-ordering half of the rule is pinned in
|
||||
/// <see cref="ClusterActiveNodeTests"/> against a real two-node cluster.
|
||||
/// </summary>
|
||||
public sealed class ActiveRoleResolutionTests
|
||||
{
|
||||
[Fact]
|
||||
public void EmptyPreference_IsUnscoped()
|
||||
{
|
||||
// Unscoped: every Up member competes for one active slot (the ScadaBridge central pattern).
|
||||
Assert.Null(ClusterActiveNode.ResolveActiveRole(["admin", "driver"], []));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FusedNode_AnswersForTheHighestPrecedenceRoleItCarries()
|
||||
{
|
||||
Assert.Equal(
|
||||
"admin",
|
||||
ClusterActiveNode.ResolveActiveRole(["admin", "driver", "cluster-MAIN"], ["admin", "driver"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SingleRoleNode_AnswersForThatRole()
|
||||
{
|
||||
// The regression the OtOpcUa fix existed for: under the old role-filtered check every
|
||||
// driver-only node reported Healthy — "or not a role member" — so all of them called
|
||||
// themselves active and no consumer could find the active node of a site cluster. lmxopcua#494.
|
||||
Assert.Equal(
|
||||
"driver",
|
||||
ClusterActiveNode.ResolveActiveRole(["driver", "cluster-SITE-A"], ["admin", "driver"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PreferenceOrderWins_NotTheOrderTheNodeListsItsRoles()
|
||||
{
|
||||
// Self-role order is incidental (it comes from config parsing); precedence must come from the
|
||||
// caller's list alone, or a fused node's answer would depend on how its roles were spelled.
|
||||
Assert.Equal(
|
||||
"admin",
|
||||
ClusterActiveNode.ResolveActiveRole(["driver", "admin"], ["admin", "driver"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NodeCarryingNoneOfThePreferredRoles_ResolvesToNull()
|
||||
{
|
||||
// Reachable, not merely defensive: OtOpcUa's RoleParser admits a dev-only or cluster-role-only
|
||||
// node. Such a node owns no active work — the health check maps this to Unhealthy so it stays
|
||||
// out of the active pool.
|
||||
Assert.Null(ClusterActiveNode.ResolveActiveRole(["dev", "cluster-MAIN"], ["admin", "driver"]));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using ZB.MOM.WW.Health.Akka;
|
||||
|
||||
namespace ZB.MOM.WW.Health.Akka.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Covers the cluster-view <c>data</c> dictionary that <see cref="AkkaClusterHealthCheck"/> publishes
|
||||
/// from 0.2.0 on (consumed by the family overview dashboard to show each cluster's current leader).
|
||||
/// Uses a real single-node self-joined cluster via <see cref="TestKit"/> — the values come off
|
||||
/// <see cref="Cluster"/>, which cannot be faked, so a formed cluster is the only honest fixture.
|
||||
/// </summary>
|
||||
public sealed class AkkaClusterDataTests : TestKit
|
||||
{
|
||||
private const string ClusterConfig = """
|
||||
akka {
|
||||
actor.provider = cluster
|
||||
remote.dot-netty.tcp {
|
||||
hostname = "127.0.0.1"
|
||||
port = 0
|
||||
}
|
||||
cluster {
|
||||
roles = ["driver", "admin"]
|
||||
# Keep the single-node self-join deterministic and fast in-test.
|
||||
min-nr-of-members = 1
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
private static readonly TimeSpan FormationTimeout = TimeSpan.FromSeconds(30);
|
||||
|
||||
public AkkaClusterDataTests()
|
||||
: base(ClusterConfig, nameof(AkkaClusterDataTests))
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Self-joins the test node and waits until it is Up with a converged leader.</summary>
|
||||
private Cluster FormSingleNodeCluster()
|
||||
{
|
||||
var cluster = Cluster.Get(Sys);
|
||||
cluster.Join(cluster.SelfAddress);
|
||||
AwaitCondition(
|
||||
() => cluster.SelfMember.Status == MemberStatus.Up && cluster.State.Leader is not null,
|
||||
FormationTimeout);
|
||||
return cluster;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildClusterData_SingleNodeCluster_ReportsSelfAsLeader()
|
||||
{
|
||||
var cluster = FormSingleNodeCluster();
|
||||
|
||||
var data = AkkaClusterHealthCheck.BuildClusterData(cluster);
|
||||
|
||||
Assert.Equal(cluster.SelfAddress.ToString(), Assert.IsType<string>(data["leader"]));
|
||||
Assert.Equal(cluster.SelfAddress.ToString(), Assert.IsType<string>(data["selfAddress"]));
|
||||
Assert.Equal(1, Assert.IsType<int>(data["memberCount"]));
|
||||
Assert.Equal(0, Assert.IsType<int>(data["unreachableCount"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildClusterData_RolesRoundTripSorted()
|
||||
{
|
||||
var cluster = FormSingleNodeCluster();
|
||||
|
||||
var roles = Assert.IsType<string[]>(AkkaClusterHealthCheck.BuildClusterData(cluster)["selfRoles"]);
|
||||
|
||||
// Sorted, so the payload is stable across restarts; the configured roles survive verbatim.
|
||||
Assert.Equal(new[] { "admin", "driver" }, roles);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CheckHealth_FormedCluster_HealthyResultCarriesClusterData()
|
||||
{
|
||||
var cluster = FormSingleNodeCluster();
|
||||
var provider = new ServiceCollection().AddSingleton(Sys).BuildServiceProvider();
|
||||
var check = new AkkaClusterHealthCheck(provider, AkkaClusterStatusPolicy.Default);
|
||||
|
||||
var result = await check.CheckHealthAsync(NewContext(check));
|
||||
|
||||
Assert.Equal(HealthStatus.Healthy, result.Status);
|
||||
Assert.Equal(cluster.SelfAddress.ToString(), result.Data["leader"]);
|
||||
Assert.Equal(1, result.Data["memberCount"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CheckHealth_NoActorSystem_CarriesNoData()
|
||||
{
|
||||
// The startup-safety paths stay description-only: an empty data dictionary is what makes the
|
||||
// writer omit the "data" key entirely, so a not-yet-started node's payload is unchanged.
|
||||
var provider = new ServiceCollection().BuildServiceProvider();
|
||||
var check = new AkkaClusterHealthCheck(provider, AkkaClusterStatusPolicy.Default);
|
||||
|
||||
var result = await check.CheckHealthAsync(NewContext(check));
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded, result.Status);
|
||||
Assert.Empty(result.Data);
|
||||
}
|
||||
|
||||
private static HealthCheckContext NewContext(IHealthCheck check) => new()
|
||||
{
|
||||
Registration = new HealthCheckRegistration("akka-cluster", check, HealthStatus.Unhealthy, tags: null),
|
||||
};
|
||||
}
|
||||
@@ -88,6 +88,8 @@ public sealed class AkkaClusterStatusPolicyTests
|
||||
var result = await check.CheckHealthAsync(NewContext(check));
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded, result.Status);
|
||||
// No cluster to describe → no data, so the emitted payload stays the pre-0.2.0 shape.
|
||||
Assert.Empty(result.Data);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using ZB.MOM.WW.Health.Akka;
|
||||
|
||||
namespace ZB.MOM.WW.Health.Akka.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins <b>which</b> node the active tier names, against a real two-node cluster deliberately built
|
||||
/// so the oldest member is <i>not</i> the lowest-addressed one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The divergence is the whole point. Akka offers two different "first" members:
|
||||
/// <c>ClusterState.Leader</c> / <c>RoleLeader</c> is the <b>lowest-addressed</b> Up member
|
||||
/// (host, then port — nothing to do with time), while <b>oldest</b> is the member with the
|
||||
/// lowest up-number, which is what <c>ClusterSingletonManager</c> uses to place singletons.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// They coincide on a freshly-formed cluster, which is why every single-node and happy-path
|
||||
/// test missed it and why the leader-based check survived to 0.2.1. They diverge the moment a
|
||||
/// node restarts: the restarted node rejoins as the youngest while keeping its address, so if
|
||||
/// it holds the lower address it becomes leader while the singletons — and every piece of work
|
||||
/// they own — stay on the other node. This fixture reproduces that state directly instead of
|
||||
/// waiting for a restart.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class ClusterActiveNodeTests : IAsyncLifetime
|
||||
{
|
||||
// Fixed, unusual ports: the test needs a deterministic address ordering, which port 0 cannot give.
|
||||
// Distinct from OtOpcUa's equivalent fixture (19_530/19_531) so the two suites can run at once.
|
||||
private const int OldestPort = 19_541; // joins FIRST -> oldest, but the HIGHER address
|
||||
private const int YoungestPort = 19_540; // joins SECOND -> cluster leader, the LOWER address
|
||||
|
||||
private const string SystemName = "health-active-node";
|
||||
|
||||
private ActorSystem? _oldest;
|
||||
private ActorSystem? _youngest;
|
||||
|
||||
private static Config NodeConfig(int port, string roles) => ConfigurationFactory.ParseString($$"""
|
||||
akka {
|
||||
loglevel = "WARNING"
|
||||
actor.provider = "Akka.Cluster.ClusterActorRefProvider, Akka.Cluster"
|
||||
remote.dot-netty.tcp {
|
||||
hostname = "127.0.0.1"
|
||||
port = {{port}}
|
||||
}
|
||||
cluster {
|
||||
seed-nodes = ["akka.tcp://{{SystemName}}@127.0.0.1:{{OldestPort}}"]
|
||||
roles = [{{roles}}]
|
||||
min-nr-of-members = 1
|
||||
run-coordinated-shutdown-when-down = off
|
||||
downing-provider-class = ""
|
||||
}
|
||||
}
|
||||
""");
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
// Order matters: the seed forms the cluster and is therefore the oldest member. The oldest
|
||||
// node is fused (admin + driver); the younger one is driver-only, so the role-scoped
|
||||
// assertions below can distinguish "oldest of this role" from "oldest overall".
|
||||
_oldest = ActorSystem.Create(SystemName, NodeConfig(OldestPort, "\"admin\", \"driver\""));
|
||||
await WaitForUpAsync(_oldest, expectedMembers: 1);
|
||||
|
||||
_youngest = ActorSystem.Create(SystemName, NodeConfig(YoungestPort, "\"driver\""));
|
||||
await WaitForUpAsync(_oldest, expectedMembers: 2);
|
||||
await WaitForUpAsync(_youngest, expectedMembers: 2);
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_youngest is not null) await _youngest.Terminate();
|
||||
if (_oldest is not null) await _oldest.Terminate();
|
||||
}
|
||||
|
||||
private static async Task WaitForUpAsync(ActorSystem system, int expectedMembers)
|
||||
{
|
||||
var cluster = Cluster.Get(system);
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expectedMembers) return;
|
||||
await Task.Delay(100);
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"cluster on port {cluster.SelfAddress.Port} never saw {expectedMembers} Up members "
|
||||
+ $"(saw {cluster.State.Members.Count(m => m.Status == MemberStatus.Up)})");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sanity check on the fixture itself: if the two orderings did not actually diverge, every
|
||||
/// assertion below would pass for the wrong reason.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Fixture_actually_produces_divergent_age_and_address_ordering()
|
||||
{
|
||||
var state = Cluster.Get(_oldest!).State;
|
||||
|
||||
Assert.Equal(YoungestPort, state.Leader!.Port);
|
||||
Assert.Equal(YoungestPort, state.RoleLeader("driver")!.Port);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OldestUpMember_IsTheOldest_NotTheLeader()
|
||||
{
|
||||
var members = Cluster.Get(_oldest!).State.Members;
|
||||
|
||||
Assert.Equal(OldestPort, ClusterActiveNode.OldestUpMember(members)!.Address.Port);
|
||||
Assert.Equal(OldestPort, ClusterActiveNode.OldestUpMember(members, "driver")!.Address.Port);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OldestUpMember_IsRoleScoped()
|
||||
{
|
||||
var members = Cluster.Get(_oldest!).State.Members;
|
||||
|
||||
// Only the fused node carries admin, so it owns the admin slot even though it is not the
|
||||
// lowest-addressed member.
|
||||
Assert.Equal(OldestPort, ClusterActiveNode.OldestUpMember(members, "admin")!.Address.Port);
|
||||
|
||||
// A role nobody carries has no owner.
|
||||
Assert.Null(ClusterActiveNode.OldestUpMember(members, "no-such-role"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelfIsActive_IsTrueOnlyOnTheOldestNode()
|
||||
{
|
||||
Assert.True(ClusterActiveNode.SelfIsActive(Cluster.Get(_oldest!)));
|
||||
Assert.False(ClusterActiveNode.SelfIsActive(Cluster.Get(_youngest!)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelfIsActive_RoleScoped_YoungestOwnsNoSlot()
|
||||
{
|
||||
// Driver-scoped: the fused node is still the oldest driver, so the driver-only node is standby.
|
||||
Assert.True(ClusterActiveNode.SelfIsActive(Cluster.Get(_oldest!), "driver"));
|
||||
Assert.False(ClusterActiveNode.SelfIsActive(Cluster.Get(_youngest!), "driver"));
|
||||
|
||||
// Admin-scoped: the driver-only node does not carry admin, so it can never be the admin active
|
||||
// node — even though it is the cluster leader.
|
||||
Assert.False(ClusterActiveNode.SelfIsActive(Cluster.Get(_youngest!), "admin"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthCheck_IsHealthyOnTheOldestAndUnhealthyOnTheLeader()
|
||||
{
|
||||
var onOldest = await CheckAsync(_oldest!, new ActiveNodeHealthCheckOptions());
|
||||
var onYoungest = await CheckAsync(_youngest!, new ActiveNodeHealthCheckOptions());
|
||||
|
||||
Assert.Equal(HealthStatus.Healthy, onOldest.Status);
|
||||
|
||||
// The node the old leader-based check would have called active. Exactly one 200 per cluster is
|
||||
// the property an orchestrator's active-tier routing depends on.
|
||||
Assert.Equal(HealthStatus.Unhealthy, onYoungest.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthCheck_DataNamesTheActiveNode_EvenFromAStandby()
|
||||
{
|
||||
// A standby reporting *who* is active is what lets a dashboard render the pair from either
|
||||
// node's payload alone (the 0.2.0 per-entry data object).
|
||||
var result = await CheckAsync(_youngest!, new ActiveNodeHealthCheckOptions());
|
||||
|
||||
Assert.Equal($"akka.tcp://{SystemName}@127.0.0.1:{OldestPort}", result.Data["activeNode"]);
|
||||
Assert.Equal($"akka.tcp://{SystemName}@127.0.0.1:{YoungestPort}", result.Data["selfAddress"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthCheck_RolePreference_FusedNodeAnswersForAdmin()
|
||||
{
|
||||
var options = new ActiveNodeHealthCheckOptions { RolePreference = ["admin", "driver"] };
|
||||
|
||||
var onOldest = await CheckAsync(_oldest!, options);
|
||||
var onYoungest = await CheckAsync(_youngest!, options);
|
||||
|
||||
Assert.Equal(HealthStatus.Healthy, onOldest.Status);
|
||||
Assert.Equal("admin", onOldest.Data["activeRole"]);
|
||||
|
||||
// Resolves to driver (it has no admin role) and loses to the older fused node.
|
||||
Assert.Equal(HealthStatus.Unhealthy, onYoungest.Status);
|
||||
Assert.Equal("driver", onYoungest.Data["activeRole"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthCheck_NodeOwningNoneOfThePreferredRoles_IsUnhealthyByDefault()
|
||||
{
|
||||
var options = new ActiveNodeHealthCheckOptions { RolePreference = ["no-such-role"] };
|
||||
|
||||
var result = await CheckAsync(_oldest!, options);
|
||||
|
||||
// Not "not applicable ⇒ Healthy": a node that owns no active work must stay out of the pool.
|
||||
Assert.Equal(HealthStatus.Unhealthy, result.Status);
|
||||
Assert.Contains("carries none of the active roles", result.Description);
|
||||
Assert.DoesNotContain("activeRole", result.Data.Keys);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthCheck_NoActiveRoleStatus_IsOverridable()
|
||||
{
|
||||
var options = new ActiveNodeHealthCheckOptions
|
||||
{
|
||||
RolePreference = ["no-such-role"],
|
||||
NoActiveRoleStatus = HealthStatus.Healthy,
|
||||
};
|
||||
|
||||
Assert.Equal(HealthStatus.Healthy, (await CheckAsync(_oldest!, options)).Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_IsActiveOnlyOnTheOldestNode()
|
||||
{
|
||||
Assert.True(new AkkaActiveNodeGate(ProviderFor(_oldest!)).IsActiveNode);
|
||||
Assert.False(new AkkaActiveNodeGate(ProviderFor(_youngest!)).IsActiveNode);
|
||||
|
||||
// Role-scoped: the driver-only node cannot gate admin work.
|
||||
Assert.False(new AkkaActiveNodeGate(ProviderFor(_youngest!), "admin").IsActiveNode);
|
||||
}
|
||||
|
||||
private static IServiceProvider ProviderFor(ActorSystem system) =>
|
||||
new ServiceCollection().AddSingleton(system).BuildServiceProvider();
|
||||
|
||||
private static async Task<HealthCheckResult> CheckAsync(
|
||||
ActorSystem system,
|
||||
ActiveNodeHealthCheckOptions options)
|
||||
{
|
||||
var check = new ActiveNodeHealthCheck(ProviderFor(system), options);
|
||||
var context = new HealthCheckContext
|
||||
{
|
||||
Registration = new HealthCheckRegistration("active-node", check, HealthStatus.Unhealthy, tags: null),
|
||||
};
|
||||
return await check.CheckHealthAsync(context);
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Akka.TestKit.Xunit2" />
|
||||
<PackageReference Include="coverlet.collector" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="xunit" />
|
||||
|
||||
@@ -22,8 +22,11 @@ public sealed class ResponseWriterTests
|
||||
{
|
||||
private readonly HealthCheckResult _result;
|
||||
|
||||
public StubHealthCheck(HealthStatus status, string? description = null) =>
|
||||
_result = new HealthCheckResult(status, description);
|
||||
public StubHealthCheck(
|
||||
HealthStatus status,
|
||||
string? description = null,
|
||||
IReadOnlyDictionary<string, object>? data = null) =>
|
||||
_result = new HealthCheckResult(status, description, exception: null, data: data);
|
||||
|
||||
public Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
@@ -31,12 +34,14 @@ public sealed class ResponseWriterTests
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseMessage> GetReadyAsync(
|
||||
HealthStatus status, string? description = "db reachable")
|
||||
HealthStatus status,
|
||||
string? description = "db reachable",
|
||||
IReadOnlyDictionary<string, object>? data = null)
|
||||
{
|
||||
var builder = WebApplication.CreateBuilder();
|
||||
builder.WebHost.UseTestServer();
|
||||
builder.Services.AddHealthChecks()
|
||||
.AddCheck("db", new StubHealthCheck(status, description), tags: new[] { ZbHealthTags.Ready });
|
||||
.AddCheck("db", new StubHealthCheck(status, description, data), tags: new[] { ZbHealthTags.Ready });
|
||||
|
||||
await using var app = builder.Build();
|
||||
app.MapZbHealth();
|
||||
@@ -84,6 +89,51 @@ public sealed class ResponseWriterTests
|
||||
Assert.Equal(JsonValueKind.Null, description.ValueKind);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadyEndpoint_EntryWithData_EmitsDataObject_WithVerbatimKeys()
|
||||
{
|
||||
// The envelope properties are camelCased by the naming policy, but a check's own data keys are
|
||||
// dictionary keys and must survive verbatim — the dashboard reads them by exact name
|
||||
// (entries["akka-cluster"].data.leader), so a naming policy applied here would silently break
|
||||
// the contract. Values may be any JSON-friendly type (string / int / string[]).
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
["leader"] = "akka.tcp://sys@node-1:4053",
|
||||
["memberCount"] = 2,
|
||||
["selfRoles"] = new[] { "driver", "site-a" },
|
||||
};
|
||||
|
||||
var response = await GetReadyAsync(HealthStatus.Healthy, data: data);
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
|
||||
using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
|
||||
var entryData = doc.RootElement.GetProperty("entries").GetProperty("db").GetProperty("data");
|
||||
|
||||
Assert.Equal("akka.tcp://sys@node-1:4053", entryData.GetProperty("leader").GetString());
|
||||
Assert.Equal(2, entryData.GetProperty("memberCount").GetInt32());
|
||||
Assert.Equal(
|
||||
new[] { "driver", "site-a" },
|
||||
entryData.GetProperty("selfRoles").EnumerateArray().Select(static e => e.GetString()).ToArray());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadyEndpoint_EntryWithoutData_OmitsDataKeyEntirely()
|
||||
{
|
||||
// The 0.2.0 non-breaking invariant: a check that publishes no data must produce exactly the
|
||||
// pre-0.2.0 body. Asserted on the raw JSON string so a `"data": null` regression (which
|
||||
// TryGetProperty would still accept as "present") fails here.
|
||||
var response = await GetReadyAsync(HealthStatus.Healthy);
|
||||
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
|
||||
Assert.DoesNotContain("\"data\"", body, StringComparison.Ordinal);
|
||||
|
||||
using var doc = JsonDocument.Parse(body);
|
||||
var db = doc.RootElement.GetProperty("entries").GetProperty("db");
|
||||
Assert.False(db.TryGetProperty("data", out _), "data key must be absent when the check set none");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadyEndpoint_Degraded_Returns200_WithDegradedStatus()
|
||||
{
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Version>0.1.3</Version>
|
||||
<Version>0.3.3</Version>
|
||||
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<!-- NU1510 (package-pruning advice): the Replication and Tests projects carry a
|
||||
|
||||
+188
-18
@@ -10,16 +10,127 @@ Change capture uses SQLite triggers writing a single HLC-stamped oplog inside th
|
||||
consumer's own write transaction (crash-consistent, non-bypassable). Convergence is
|
||||
last-writer-wins over a hybrid logical clock. All timestamps are UTC.
|
||||
|
||||
## Packages (all `0.1.3`)
|
||||
## Packages (all `0.3.3`)
|
||||
|
||||
| Package | Contents |
|
||||
|---|---|
|
||||
| `ZB.MOM.WW.LocalDb` | Embedded DB host: `ILocalDb` SQL surface (`ExecuteAsync` / `QueryAsync` / transactions), `RegisterReplicated(table)` (PK validation + trigger install + oplog schema), `HybridLogicalClock`, `AddZbLocalDb()` DI. |
|
||||
| `ZB.MOM.WW.LocalDb` | Embedded DB host: `ILocalDb` SQL surface (`ExecuteAsync` / `QueryAsync` / transactions), `RegisterReplicated(table[, baselineExistingRows])` (PK validation + trigger install + oplog schema), `DeregisterReplicated(table)` (trigger removal + oplog/ledger prune), `HybridLogicalClock`, `AddZbLocalDb()` DI. |
|
||||
| `ZB.MOM.WW.LocalDb.Contracts` | `localdb_sync.v1` proto + generated gRPC types (packable; a future peer could be non-.NET). |
|
||||
| `ZB.MOM.WW.LocalDb.Replication` | gRPC sync engine: passive `LocalDbSyncService`, initiating `SyncBackgroundService`, `MaintenanceBackgroundService`, LWW apply, watermark acks + pruning, snapshot resync, `ISyncStatus`, `LocalDbMetrics`, `AddZbLocalDbReplication()` / `MapZbLocalDbSync()`. |
|
||||
|
||||
> Status: **published to the Gitea feed; adopted by ScadaBridge (Phases 1+2) and OtOpcUa (Phase 1).**
|
||||
>
|
||||
> - `0.3.3` (2026-08-19) — one Low finding from the ScadaBridge adversarial architecture review
|
||||
> (P4-7), in two halves. Wire-compatible with every earlier peer: no proto change, no
|
||||
> `lib_schema_version` bump, no on-disk schema change.
|
||||
> - **`RegisterReplicated` now refuses a table that another table cascades a DELETE onto.**
|
||||
> Applying a remote row uses `INSERT OR REPLACE`, whose implicit DELETE fires `ON DELETE
|
||||
> CASCADE` / `SET NULL` / `SET DEFAULT` on every child — while the apply path holds the capture
|
||||
> guard raised, so the child rows it touches produce no oplog entry, the peer keeps them, and the
|
||||
> pair diverges with nothing reporting a fault. Registration is the only place that is visible,
|
||||
> so it is refused there with an actionable `LocalDbRegistrationException` naming the referencing
|
||||
> table and its action. `NO ACTION` / `RESTRICT` are still allowed: they mutate no child row (the
|
||||
> implicit delete simply fails, which the applier dead-letters — loud, not silent). No current
|
||||
> consumer has a foreign key on a replicated table, so this is a guard against a future foot-gun
|
||||
> rather than a live fix.
|
||||
> - **Applying one entry is atomic across its row write and its ledger write.** They were two
|
||||
> statements under the poison catch, so a failure landing between them committed a
|
||||
> deleted/replaced row carrying a STALE LIVE ledger version — the shape `SnapshotStreamer` warns
|
||||
> about and skips, meaning that key never travels again. A per-entry `SAVEPOINT` makes the pair
|
||||
> all-or-nothing while keeping the catch's per-ENTRY granularity: entries applied earlier in the
|
||||
> same batch survive, and the poison entry is still dead-lettered and the batch still continues.
|
||||
> - `0.3.2` (2026-08-19) — two Medium findings from the ScadaBridge adversarial architecture review
|
||||
> (P4-1, P4-3). Wire-compatible with every earlier peer: no proto change, no `lib_schema_version`
|
||||
> bump, no on-disk schema change — the keepalive is transport-level and the rest is local policy.
|
||||
> - **A live session now ends itself when a cap prune eats deltas it has not sent.** The pump reads
|
||||
> `seq > sentThruSeq`, which resumes SILENTLY above a pruned hole; the peer acks the later seqs
|
||||
> and the pair is durably divergent for the whole life of that session. `needs_snapshot` (set in
|
||||
> the prune's own transaction) only heals a RECONNECT, and nothing forced one — so on a stable
|
||||
> LAN with a slow-but-alive peer the divergence was unbounded. Every pump iteration now compares
|
||||
> the oplog horizon (`MIN(seq)`, one b-tree seek) against its sent watermark and throws
|
||||
> `LocalDbOplogGapException` when deltas it still owes are gone. The horizon, not the flag,
|
||||
> because the flag is also raised for reasons that do not invalidate the running session.
|
||||
> - **The sync stream has HTTP/2 keepalive, and the server's single-stream guard is newest-wins.**
|
||||
> Neither end ever timed out on silence (a converged pair is legitimately quiet), so a hard crash
|
||||
> or partition of the dialling node left the passive node's handler parked in `MoveNext` on a
|
||||
> half-open stream — holding the guard, reporting itself connected, and rejecting the rebooted
|
||||
> peer's reconnects with `AlreadyExists`, indefinitely on an idle pair. The initiator now dials
|
||||
> with `KeepAlivePingDelay` / `KeepAlivePingTimeout` (new options, 60s/20s, `Always` policy so
|
||||
> idle connections ping too), and a fresh authenticated `Sync` cancels the incumbent and waits
|
||||
> for it to unwind before starting — the guard serializes sessions, it was never a fence, and an
|
||||
> authenticated same-key dialler is by definition the peer. An incumbent that will not yield
|
||||
> within the takeover timeout still gets `AlreadyExists`, because two live sessions over one
|
||||
> database is worse than a refused dial.
|
||||
> - `0.3.1` (2026-08-18) — a faulting session flushes its control lane before cancelling its writer,
|
||||
> so a locally-detected handshake mismatch actually reaches the peer instead of leaving it to
|
||||
> reconnect-loop against a generic disconnect.
|
||||
> - `0.3.0` (2026-08-15) — **the snapshot-resync flag is per-table.** On-disk bookkeeping schema
|
||||
> goes to **v3** (new `__localdb_snapshot_state`, upgraded in place on open); wire-compatible with
|
||||
> `0.1.x`/`0.2.x` peers, and `lib_schema_version` deliberately stays `1` because it is compared
|
||||
> fail-closed for equality.
|
||||
> - **`RegisterReplicated(table, baselineExistingRows: true)` now flags a resync for THAT TABLE.**
|
||||
> Through `0.2.x` the flag was one bit for the whole database, so baselining one table
|
||||
> re-streamed every registered table in full — and on a pair enabling replication late, both
|
||||
> nodes did that to each other at once. A snapshot now streams only the flagged tables.
|
||||
> - **The database-wide flag survives, unchanged, for the cases that genuinely need it**: an oplog
|
||||
> cap prune or an impossible peer ack loses deltas for the oplog as a whole, so nothing short of
|
||||
> every table is sound. What moved is only where a *baseline* records its debt.
|
||||
> - **Additive wire extension, negotiated by capability, not by version.** `Handshake` gains
|
||||
> `supports_partial_snapshot`; `SnapshotBegin` gains a `tables` scope where **empty means "all
|
||||
> tables"** — which is precisely what a pre-`0.3.0` sender emits and what a pre-`0.3.0` receiver
|
||||
> assumes. A peer that does not advertise the capability is served a **full** snapshot, because a
|
||||
> scoped one would make it advance `last_applied_remote_seq` past deltas it never received.
|
||||
> - **A partial snapshot moves no watermark on either side.** The receiver keeps
|
||||
> `last_applied_remote_seq` (it is one number for the whole peer oplog, and the untouched tables'
|
||||
> deltas still sit below the snapshot's as-of); the sender keeps pumping from where it was. The
|
||||
> overlap re-delivers rows the snapshot already carried, which LWW absorbs.
|
||||
> - **Snapshot debt now clears on acknowledgement, not on enqueue** (new `SnapshotAck`, exchanged
|
||||
> only between `0.3.0` peers). A crash part-way through a snapshot re-streams the same scope on
|
||||
> reconnect instead of losing it. Against a pre-`0.3.0` peer the `0.2.x` clear-on-enqueue
|
||||
> behaviour is kept — there is no ack to wait for.
|
||||
> - `0.2.1` (2026-08-14) — two adversarial-review findings against `0.2.0`. Wire-compatible with
|
||||
> `0.1.x`/`0.2.0` peers; no proto change, no `lib_schema_version` bump, no on-disk schema change.
|
||||
> - **`DeregisterReplicated` no longer leaves the HLC without a crash anchor.** Clock recovery on
|
||||
> open is `MAX(hlc_high_water, oplog HLCs, ledger HLCs, last_seen_hlc)`, and `hlc_high_water` was
|
||||
> written only by graceful disposal — while deregistration deletes the table's oplog AND ledger
|
||||
> rows. A crash after a deregister boot could therefore recover the clock below stamps the peer
|
||||
> already holds, and LWW discards those as older. The current clock is now flushed inside the
|
||||
> deregistration transaction, before the prune (and the flush is MAX-guarded, so it can never
|
||||
> regress the anchor).
|
||||
> - **The sync inbox is bounded (64 messages) instead of unbounded**, so inbound memory is capped
|
||||
> at roughly `64 x MaxBatchBytes` rather than "whatever the peer sends" — the bad case being a
|
||||
> mutual full snapshot of a large dataset, buffered whole on both nodes at once. A full inbox now
|
||||
> stops draining the gRPC stream, which closes the peer's flow-control window.
|
||||
> - **A snapshot this node owes is streamed alongside a running receive loop**, not from inside the
|
||||
> handshake. That is what makes bounding safe: previously neither side drained its inbox while
|
||||
> sending, so a mutual snapshot converged only because the inbox was unbounded. Interleaving was
|
||||
> already safe (snapshot rows and deltas both merge through LWW, the watermark write is
|
||||
> MAX-guarded) — only the ordering changed.
|
||||
> - **Documented the cost of late baselining**: the snapshot flag is per-database, so baselining
|
||||
> one table re-streams every registered table, in both directions on a late-enabled pair. Trim
|
||||
> high-volume tables (event logs) before turning replication on late.
|
||||
> - `0.2.0` (2026-08-14) — performance + lifecycle work from the ScadaBridge architectural review.
|
||||
> All of it is wire-compatible with `0.1.x` peers (no proto change, no `lib_schema_version` bump);
|
||||
> the on-disk bookkeeping schema goes to **v2** and is upgraded in place on open.
|
||||
> - **`DeregisterReplicated(table)`** — drops a table's capture triggers and prunes its oplog and
|
||||
> row-version rows. A node that does not replicate should not be paying for capture on every
|
||||
> write, and until now there was no way to stop once a file had been registered. Idempotent, and
|
||||
> usable on triggers left behind by an earlier build.
|
||||
> - **`RegisterReplicated(table, baselineExistingRows: true)`** — closes the late-opt-in hole: rows
|
||||
> that predate registration were invisible to the peer forever, because the snapshot streamer
|
||||
> pages from the row-version ledger and CDC never wrote them there. Opting in seeds the ledger at
|
||||
> an LWW-safe floor (HLC `0`, this node's id) and flags a snapshot resync so they actually
|
||||
> travel. Default is still off — the documented CDC contract is unchanged unless you ask.
|
||||
> - **O(1) backlog depth** — `MAX(seq) - MAX(last_acked_seq, MIN(seq) - 1)` replaces a
|
||||
> `COUNT(*)` over the whole unacked backlog that ran once a minute AND on every metrics scrape
|
||||
> (measured 58 VM steps vs 100,020 on a 50k backlog).
|
||||
> - **Byte-budget batching** — `MaxBatchBytes` (default 2 MB) bounds delta and snapshot messages by
|
||||
> summed serialized size, with `MaxBatchSize` demoted to a secondary row cap. Consumers no longer
|
||||
> have to tune the row count against their widest row to stay under the gRPC message limit.
|
||||
> Inbound delta batches and acks already queued are coalesced into one transaction / one ack.
|
||||
> - **`LwwApplier` prepares once per statement** instead of three SQL parses per applied row.
|
||||
> - **Dropped `__localdb_oplog_hlc`** — an index no query ever used, costing a b-tree insert inside
|
||||
> every captured consumer write. Existing files lose it on the next open.
|
||||
> - `0.1.3` (2026-07-21) — fixes the other half of the rebuilt-peer story: a rebuilt peer's OWN
|
||||
> writes were silently dropped. `last_applied_remote_seq` is a watermark in the *peer's* seq
|
||||
> space, and a rebuilt peer numbers from 1 again — so the healthy node advertised the old peer's
|
||||
@@ -107,7 +218,8 @@ node still pushes its own deltas back. Only one side needs `PeerAddress`.
|
||||
| `PeerAddress` | `""` | gRPC address to dial. Empty ⇒ passive (initiator idles). |
|
||||
| `ApiKey` | `null` | Bearer token the initiator presents on its channel; `null` ⇒ no key header. |
|
||||
| `FlushInterval` | `250 ms` | How often the change pump flushes pending oplog deltas. |
|
||||
| `MaxBatchSize` | `500` | Max oplog entries per outbound batch (also the snapshot page size). |
|
||||
| `MaxBatchBytes` | `2 MB` | **Primary** batch cap: summed serialized bytes per outbound delta/snapshot message. Sized under the 4 MB gRPC default; a single row above the budget is still sent, alone. |
|
||||
| `MaxBatchSize` | `500` | **Secondary** row-count cap per outbound batch (also the snapshot DB page size). |
|
||||
| `MaxOplogRows` | `1_000_000` | Backlog ceiling; exceeding it flags a snapshot resync and prunes oldest rows. |
|
||||
| `MaxOplogAge` | `7 days` | Age ceiling; an unpruned entry older than this flags a snapshot resync. |
|
||||
| `TombstoneRetention` | `7 days` | How long deleted-row tombstones are kept before pruning. |
|
||||
@@ -133,8 +245,15 @@ All options are validated at startup (`ZB.MOM.WW.Configuration` validators).
|
||||
delta/snapshot batches), so receiving never blocks on the ability to ack.
|
||||
- **Resync** — when a peer's watermark falls below the pruned oplog horizon, it is sent a
|
||||
full snapshot of the registered tables (streamed, keyset-paged) and deltas resume above
|
||||
the snapshot's as-of seq. **Each node decides from its OWN state** whether it owes the
|
||||
peer a snapshot.
|
||||
the snapshot's as-of seq. When instead a single table was *baselined*, only that table is
|
||||
streamed and **no watermark moves on either side** — a partial snapshot speaks for nothing
|
||||
outside its own tables, so the deltas it overlaps are still pumped and merged by LWW.
|
||||
**Each node decides from its OWN state** whether it owes the peer a snapshot, and a peer
|
||||
that does not advertise `supports_partial_snapshot` is sent a full one. The snapshot is
|
||||
streamed *after* the handshake, alongside a running receive loop, so a node sending one
|
||||
still applies what the peer sends it — a mutual snapshot exchanges under flow control
|
||||
instead of being buffered whole in memory. Between `0.3.0` peers the sender's flags are
|
||||
cleared by the receiver's `SnapshotAck`, so a crash mid-snapshot re-streams it.
|
||||
- **Maintenance** — a background service on **both** roles enforces the oplog row/age caps
|
||||
and prunes expired tombstones every `MaintenanceInterval`.
|
||||
|
||||
@@ -160,14 +279,41 @@ All options are validated at startup (`ZB.MOM.WW.Configuration` validators).
|
||||
`RegisterReplicated`, because `json_object` cannot capture it). A *typeless* column can
|
||||
still hold a BLOB value at runtime, which then surfaces as a write-time error, not a
|
||||
registration error.
|
||||
- **Replicated tables must not be cascade-delete foreign-key parents.** If another table
|
||||
references the one you are registering with `ON DELETE CASCADE` (or `SET NULL` /
|
||||
`SET DEFAULT`), `RegisterReplicated` throws. Applying a remote row uses `INSERT OR
|
||||
REPLACE`, whose implicit DELETE fires that referential action while change capture is
|
||||
suppressed — so the child rows it deletes or nulls are never captured, the peer keeps
|
||||
them, and the pair diverges with nothing reporting a fault. Model the parent/child
|
||||
relationship with `ON DELETE NO ACTION` (or `RESTRICT`) and delete children explicitly, or
|
||||
leave the parent unreplicated. `NO ACTION` / `RESTRICT` are allowed because they mutate no
|
||||
child row: the implicit delete just fails, and the applier dead-letters the entry — loud,
|
||||
not silent. The check runs at **registration time**, so a cascading child table created
|
||||
*later* is not caught (SQLite offers no DDL hook); register after schema setup, which is
|
||||
where startup does it anyway.
|
||||
- **CDC semantics** — rows that existed **before** `RegisterReplicated` are not captured
|
||||
and are not snapshotted; only changes after registration are replicated. Re-baselining
|
||||
pre-existing rows requires re-writing them.
|
||||
and are not snapshotted; only changes after registration are replicated. **Opt out with
|
||||
`RegisterReplicated(table, baselineExistingRows: true)`**, which seeds the row-version
|
||||
ledger for existing rows at the baseline floor (HLC `0`) and flags a snapshot resync
|
||||
**for that table** so they reach the peer. The floor is below every genuine stamp, so a baselined row loses
|
||||
every comparison against a real remote write and wins only where the peer has no version
|
||||
of that key at all. Two nodes that baseline the same key both hold HLC `0` and resolve by
|
||||
the node-id tie-break — convergent, but arbitrary as to which content survives.
|
||||
- **A snapshot still ships whole tables.** The flag is per-table as of `0.3.0`, so baselining
|
||||
one table streams that table and no other — but *that* table travels in full, and two
|
||||
cases still stream **everything**: an oplog cap prune / impossible peer ack (delta
|
||||
continuity is lost for the oplog as a whole), and any peer older than `0.3.0` (which
|
||||
cannot be told a snapshot is partial). So **purge or trim high-volume tables such as event
|
||||
logs before baselining them**, and expect one slow first sync. Bounded either way (batch
|
||||
budget out, bounded inbox in).
|
||||
- **REAL ±Infinity** is carried in SQLite's JSON dialect on the wire (fine for the
|
||||
SQLite↔SQLite peers this library targets).
|
||||
- `AddZbLocalDb` / `AddZbLocalDbReplication` are **first-registration-wins** (idempotent).
|
||||
- A **mutual simultaneous snapshot** buffers the peer's inbound snapshot in memory while
|
||||
this node is still sending its own (acceptable for the small datasets this lib targets).
|
||||
- **Inbound buffering is bounded, not zero.** The duplex inbox holds up to 64 messages, so
|
||||
peak inbound memory is roughly `64 x MaxBatchBytes` (~128 MB at the 2 MB default, far
|
||||
less in practice — messages are only that large when the sender has that much backlog).
|
||||
Lower `MaxBatchBytes` to lower the ceiling. Beyond the bound the stream stops being
|
||||
drained and the peer is slowed by flow control.
|
||||
|
||||
## Observability
|
||||
|
||||
@@ -196,8 +342,28 @@ provider is wired or the poll threw, so a DB failure never reads as "0 backlog")
|
||||
|
||||
## Ops notes
|
||||
|
||||
- When the oplog row/age caps are exceeded, maintenance prunes to the ceiling and flags
|
||||
`needs_snapshot` (disk safety over delta continuity); the peer then snapshot-resyncs.
|
||||
- When the oplog row/age caps are exceeded, maintenance prunes to the ceiling and flags the
|
||||
**database-wide** `needs_snapshot` (disk safety over delta continuity); the peer then gets a
|
||||
full snapshot. Per-table debt lives separately in `__localdb_snapshot_state` and is what
|
||||
baselining writes.
|
||||
- **Upgrading a pair to `0.3.0`** needs no coordination: the two nodes negotiate scoped snapshots
|
||||
by capability, so a `0.3.0` node paired with a `0.2.x` one simply keeps sending full snapshots
|
||||
until both are upgraded. Rolling the file forward is one-way — a `0.2.x` build reopening a v3
|
||||
file leaves `__localdb_snapshot_state` in place and ignores it, so any per-table debt recorded
|
||||
by `0.3.0` is invisible to it (roll forward again, or re-baseline).
|
||||
- **Turning a table off** — `DeregisterReplicated(table)` drops its capture triggers and prunes
|
||||
its oplog + ledger rows. It must be applied **symmetrically**: the handshake compares
|
||||
registered-table digests fail-closed, so a node that drops a table its peer still
|
||||
replicates stops syncing with a schema-mismatch error. Deregister on both nodes, or on
|
||||
neither. Re-enabling later should pass `baselineExistingRows: true`, or the rows that
|
||||
accumulated while capture was off stay invisible to the peer — but trim high-volume
|
||||
tables first (see Limitations: baselining flags a full, all-tables snapshot). The HLC's
|
||||
durable anchor is flushed in the same transaction, so a crash after a deregister boot
|
||||
cannot bring the clock back below stamps peers already hold.
|
||||
- **On-disk schema upgrades** are applied in place when the file is opened, in one
|
||||
transaction, and every step is safe to re-run. `schema_version` in `__localdb_meta` is
|
||||
local-file state only — it is NOT the wire `lib_schema_version`, so a pair mid-upgrade
|
||||
keeps syncing.
|
||||
- The **`__localdb_dead_letter`** table holds poison rows (bad JSON, null-row non-tombstone
|
||||
entries) and over-drift rejects (when `FailClosedOnDrift`). Inspect it to diagnose
|
||||
rejects.
|
||||
@@ -208,13 +374,17 @@ provider is wired or the poll threw, so a DB failure never reads as "0 backlog")
|
||||
|
||||
## Testing
|
||||
|
||||
144 tests, all offline / no live dependencies — unit (HLC ordering + restart
|
||||
persistence, trigger SQL generation, LWW decision table, watermark/pruning, digest
|
||||
computation), integration over two real SQLite DBs on an in-memory (and real) gRPC pipe
|
||||
(bidirectional convergence, delete/update races, offline-accumulate → reconnect,
|
||||
prune → snapshot resync), plus seeded randomized property tests (random interleaved op
|
||||
sequences → both DBs identical after quiesce). Family conventions apply
|
||||
(`TreatWarningsAsErrors`).
|
||||
199 tests, all offline / no live dependencies — unit (HLC ordering + restart
|
||||
persistence, deregistration's HLC anchor surviving an ungraceful reopen, trigger SQL
|
||||
generation, LWW decision table, watermark/pruning, digest computation, schema-version
|
||||
upgrade, deregistration idempotency, baseline-floor LWW safety, O(1) depth vs a seeded
|
||||
backlog, byte-budget chunking, bounded-inbox backpressure under a slow reader), integration
|
||||
over two real SQLite DBs on an in-memory (and real) gRPC pipe (bidirectional convergence,
|
||||
delete/update races, offline-accumulate → reconnect, prune → snapshot resync, mutual
|
||||
snapshot over a capacity-1 transport, single-table baseline streaming only that table,
|
||||
mixed-version interop where a peer without `supports_partial_snapshot` gets a full one,
|
||||
snapshot debt surviving until the peer's `SnapshotAck`), plus seeded randomized property tests (random interleaved op sequences → both DBs identical after quiesce).
|
||||
Family conventions apply (`TreatWarningsAsErrors`).
|
||||
|
||||
Build / test from this directory:
|
||||
|
||||
|
||||
@@ -16,9 +16,19 @@ set -euo pipefail
|
||||
: "${GITEA_NUGET_SOURCE:?set GITEA_NUGET_SOURCE to your Gitea NuGet feed URL}"
|
||||
: "${GITEA_NUGET_KEY:?set GITEA_NUGET_KEY to your Gitea access token}"
|
||||
|
||||
dotnet pack -c Release -o ./artifacts
|
||||
# Named solution, named output, and a cd to the repo root, all for one reason: `dotnet pack`
|
||||
# with neither a project argument nor a fixed working directory packs whatever solution
|
||||
# happens to sit in the CALLER's directory — and then pushes it to a live feed under this
|
||||
# script's name. Invoked as `bash /path/to/ZB.MOM.WW.LocalDb/build/push.sh` from another
|
||||
# repo's checkout, the relative form packed and published 25 unrelated packages (recovered
|
||||
# with `dotnet nuget delete`, but only because the token happened to carry delete rights).
|
||||
# A publish script must be unambiguous about WHAT it publishes no matter where it is run.
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
dotnet nuget push "./artifacts/*.nupkg" \
|
||||
dotnet pack ZB.MOM.WW.LocalDb.slnx -c Release -o "$REPO_ROOT/artifacts"
|
||||
|
||||
dotnet nuget push "$REPO_ROOT/artifacts/*.nupkg" \
|
||||
--source "$GITEA_NUGET_SOURCE" \
|
||||
--api-key "$GITEA_NUGET_KEY" \
|
||||
--skip-duplicate
|
||||
|
||||
@@ -16,6 +16,7 @@ message SyncMessage {
|
||||
SnapshotBegin snapshot_begin = 5; // sender switches to snapshot mode
|
||||
SnapshotBatch snapshot_batch = 6;
|
||||
SnapshotComplete snapshot_complete = 7;
|
||||
SnapshotAck snapshot_ack = 8; // 0.3.0+; only sent when the peer advertised support
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +25,13 @@ message Handshake {
|
||||
uint32 lib_schema_version = 2;
|
||||
repeated TableDigest tables = 3; // fail-closed on any mismatch
|
||||
int64 last_applied_remote_seq = 4; // highest of YOUR oplog seqs I have applied
|
||||
// 0.3.0+ capability, additive: I understand a snapshot scoped to a SUBSET of the registered
|
||||
// tables (SnapshotBegin.tables) and I will answer a completed snapshot with a SnapshotAck.
|
||||
// A pre-0.3.0 peer leaves this false — proto3 default — and a 0.3.0 sender then promotes any
|
||||
// scoped snapshot to a full one and clears its flags on enqueue, exactly as 0.2.x did. This is
|
||||
// why lib_schema_version stays 1: it is compared fail-closed for EQUALITY, so bumping it would
|
||||
// stop 0.1.x/0.2.x pairs syncing outright.
|
||||
bool supports_partial_snapshot = 5;
|
||||
}
|
||||
message TableDigest { string table_name = 1; string digest = 2; }
|
||||
message HandshakeAck { string node_id = 1; bool snapshot_required = 2; }
|
||||
@@ -40,7 +48,12 @@ message OplogEntry {
|
||||
message DeltaBatch { repeated OplogEntry entries = 1; }
|
||||
message DeltaAck { int64 applied_thru_seq = 1; }
|
||||
|
||||
message SnapshotBegin { int64 as_of_seq = 1; }
|
||||
// tables: the SCOPE of this snapshot (0.3.0+). EMPTY means every registered table — a full
|
||||
// snapshot, which is the only thing a pre-0.3.0 sender ever emits and the only thing a
|
||||
// pre-0.3.0 receiver can safely assume. A non-empty list means the snapshot is PARTIAL, so the
|
||||
// receiver must NOT advance last_applied_remote_seq to as_of_seq (deltas below it for the tables
|
||||
// NOT listed have not been delivered); it merges the rows through LWW and leaves the watermark.
|
||||
message SnapshotBegin { int64 as_of_seq = 1; repeated string tables = 2; }
|
||||
message SnapshotBatch { string table_name = 1; repeated SnapshotRow rows = 2; }
|
||||
message SnapshotRow {
|
||||
string pk_json = 1;
|
||||
@@ -50,3 +63,8 @@ message SnapshotRow {
|
||||
bool is_tombstone = 5;
|
||||
}
|
||||
message SnapshotComplete { int64 as_of_seq = 1; }
|
||||
// Sent by the RECEIVER once a snapshot has been durably applied, so the sender clears the
|
||||
// per-table snapshot flags only after the peer actually holds the rows (a crash mid-snapshot then
|
||||
// re-streams the same scope on reconnect). Only exchanged when both peers advertised
|
||||
// supports_partial_snapshot; a pre-0.3.0 pair keeps the 0.2.x clear-on-enqueue behaviour.
|
||||
message SnapshotAck { int64 as_of_seq = 1; }
|
||||
|
||||
@@ -12,19 +12,58 @@ namespace ZB.MOM.WW.LocalDb.Replication.Internal;
|
||||
internal static class GrpcSyncDuplex
|
||||
{
|
||||
/// <summary>
|
||||
/// Wires <paramref name="reader"/> into an unbounded inbox channel drained by a background loop,
|
||||
/// and exposes <paramref name="writer"/> as the duplex send. Returns the duplex plus the reader
|
||||
/// loop's task so the caller can observe it. On clean stream end the inbox channel is completed;
|
||||
/// on a reader fault it is completed WITH the exception — the session's handshake and receive
|
||||
/// phases both await the inbox, so terminating it is what unblocks them when the call dies (they
|
||||
/// would otherwise hang on a dead stream).
|
||||
/// Inbound messages buffered between the transport and the session's receive loop.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why bounded at all.</b> An unbounded inbox makes this process's memory a function of what
|
||||
/// the PEER decides to send. The worst case is not hypothetical: a full snapshot streams every
|
||||
/// row of every registered table, and both sides of a pair that enabled replication late
|
||||
/// snapshot each other simultaneously — so an unbounded inbox holds an entire peer dataset (an
|
||||
/// event log at its storage cap, say) in memory on both nodes at once. Bounding it converts that
|
||||
/// into backpressure: the pump stops draining the gRPC stream, the HTTP/2 flow-control window
|
||||
/// closes, and the SENDER slows down.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why 64.</b> Memory here is bounded by roughly capacity x
|
||||
/// <see cref="ReplicationOptions.MaxBatchBytes"/> (the sender's own per-message budget), so the
|
||||
/// ceiling is a configured number rather than a peer-controlled one. 64 is four times the
|
||||
/// sender's outbound data-lane depth, so a healthy stream never reaches the bound and pays
|
||||
/// nothing; it engages only when this node is genuinely slower than the peer for a sustained
|
||||
/// burst, which is exactly when backpressure is wanted. Lower <c>MaxBatchBytes</c> to lower the
|
||||
/// ceiling.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>What makes bounding safe.</b> Backpressure on the inbox deadlocks any design where both
|
||||
/// peers can stop draining their inbox at the same time — mutual snapshots were exactly that
|
||||
/// case while the send ran inside the handshake. <see cref="SyncSession"/> therefore starts its
|
||||
/// receive loop BEFORE streaming a snapshot it owes, so the inbox is drained continuously for
|
||||
/// the whole life of the session; see the remarks on its snapshot phase.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal const int InboxCapacity = 64;
|
||||
|
||||
/// <summary>
|
||||
/// Wires <paramref name="reader"/> into a bounded inbox channel (see
|
||||
/// <see cref="InboxCapacity"/>) drained by a background loop, and exposes
|
||||
/// <paramref name="writer"/> as the duplex send. Returns the duplex plus the reader loop's task
|
||||
/// so the caller can observe it. On clean stream end the inbox channel is completed; on a reader
|
||||
/// fault it is completed WITH the exception — the session's handshake and receive phases both
|
||||
/// await the inbox, so terminating it is what unblocks them when the call dies (they would
|
||||
/// otherwise hang on a dead stream).
|
||||
/// </summary>
|
||||
public static (SyncDuplex Duplex, Task ReaderTask) Create(
|
||||
IAsyncStreamReader<SyncMessage> reader,
|
||||
IAsyncStreamWriter<SyncMessage> writer,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var inbox = Channel.CreateUnbounded<SyncMessage>(new UnboundedChannelOptions { SingleWriter = true });
|
||||
var inbox = Channel.CreateBounded<SyncMessage>(new BoundedChannelOptions(InboxCapacity)
|
||||
{
|
||||
// Wait, never Drop: every inbound message is protocol state (a delta the peer will not
|
||||
// resend in-session, a snapshot frame, an ack). Dropping one is silent divergence.
|
||||
FullMode = BoundedChannelFullMode.Wait,
|
||||
SingleWriter = true,
|
||||
});
|
||||
var readerTask = PumpInboundAsync(reader, inbox.Writer, ct);
|
||||
var duplex = new SyncDuplex
|
||||
{
|
||||
@@ -44,7 +83,10 @@ internal static class GrpcSyncDuplex
|
||||
try
|
||||
{
|
||||
while (await reader.MoveNext(ct))
|
||||
inbox.TryWrite(reader.Current);
|
||||
// WriteAsync, not TryWrite: on a FULL bounded channel TryWrite returns false and the
|
||||
// message is silently lost. Awaiting here is the backpressure — it stops draining the
|
||||
// gRPC stream, which closes the peer's flow-control window.
|
||||
await inbox.WriteAsync(reader.Current, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -14,12 +14,28 @@ internal sealed record ApplyResult(long AppliedThruSeq, int Applied, int Discard
|
||||
/// row-version (hlc, then node_id ordinal tie-break), poison rows are dead-lettered while the
|
||||
/// batch continues, and the remote watermark advances in the SAME transaction as the data so a
|
||||
/// crash-replay of the batch is a no-op (LWW's strict-greater compare loses on equal versions).
|
||||
/// Each entry's row write and its ledger write are additionally scoped by a per-entry SAVEPOINT, so
|
||||
/// the pair is all-or-nothing even when the poison catch keeps the batch alive around it.
|
||||
/// This never writes <c>__localdb_oplog</c>: the 2-node topology does not re-forward applied deltas.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every statement is issued through a <see cref="PreparedStatements"/> set held for the life of the
|
||||
/// batch, so each distinct statement is prepared ONCE and then rebound per row. The previous shape
|
||||
/// went through the generic <c>ILocalDbTransaction</c> façade, which builds a fresh
|
||||
/// <see cref="SqliteCommand"/> per call — three SQL parses per applied row (version lookup, the
|
||||
/// table's delete/upsert, the row-version write), on the path that drains a reconnect backlog.
|
||||
/// </remarks>
|
||||
internal sealed class LwwApplier(SqliteLocalDb db, Func<DateTimeOffset>? utcNow = null)
|
||||
{
|
||||
private readonly Func<DateTimeOffset> _utcNow = utcNow ?? (() => DateTimeOffset.UtcNow);
|
||||
|
||||
/// <summary>
|
||||
/// Distinct statements prepared by the most recent <see cref="ApplyBatchAsync"/>. Test-only
|
||||
/// observation: it is the direct evidence that statement count tracks the number of distinct
|
||||
/// STATEMENTS rather than the number of rows.
|
||||
/// </summary>
|
||||
internal int LastBatchStatementCount { get; private set; }
|
||||
|
||||
public async Task<ApplyResult> ApplyBatchAsync(IReadOnlyList<OplogEntryRecord> entries, CancellationToken ct)
|
||||
{
|
||||
if (entries.Count == 0)
|
||||
@@ -42,13 +58,14 @@ internal sealed class LwwApplier(SqliteLocalDb db, Func<DateTimeOffset>? utcNow
|
||||
if (entry.Hlc > maxHlc) maxHlc = entry.Hlc;
|
||||
}
|
||||
|
||||
var sqlByTable = new Dictionary<string, (string Delete, string Upsert)>(StringComparer.Ordinal);
|
||||
var applied = 0;
|
||||
var discarded = 0;
|
||||
var deadLettered = 0;
|
||||
|
||||
await using var tx = await db.BeginTransactionAsync(ct);
|
||||
await tx.ExecuteAsync("UPDATE __localdb_applying SET applying = 1 WHERE id = 1", null, ct);
|
||||
await using var tx = await db.BeginPreparedTransactionAsync(ct);
|
||||
using var sql = new PreparedStatements(tx.Connection, tx.Sqlite);
|
||||
|
||||
await sql.SetApplyingAsync(1, ct);
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
@@ -56,19 +73,16 @@ internal sealed class LwwApplier(SqliteLocalDb db, Func<DateTimeOffset>? utcNow
|
||||
// KEY table that silently inserts a phantom auto-rowid row — so it is poison, not a win.
|
||||
if (!entry.IsTombstone && entry.RowJson is null)
|
||||
{
|
||||
await DeadLetterAsync(tx, entry, "non-tombstone entry with null row_json", utcNow, ct);
|
||||
await sql.DeadLetterAsync(entry, "non-tombstone entry with null row_json", utcNow, ct);
|
||||
deadLettered++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var local = await tx.QueryAsync(
|
||||
"SELECT hlc, node_id FROM __localdb_row_version WHERE table_name = @t AND pk_json = @pk",
|
||||
static r => (Hlc: r.GetInt64(0), NodeId: r.GetString(1)),
|
||||
new { t = entry.TableName, pk = entry.PkJson }, ct);
|
||||
var local = await sql.ReadLocalVersionAsync(entry.TableName, entry.PkJson, ct);
|
||||
|
||||
var remoteWins = local.Count == 0
|
||||
|| entry.Hlc > local[0].Hlc
|
||||
|| (entry.Hlc == local[0].Hlc && string.CompareOrdinal(entry.NodeId, local[0].NodeId) > 0);
|
||||
var remoteWins = local is null
|
||||
|| entry.Hlc > local.Value.Hlc
|
||||
|| (entry.Hlc == local.Value.Hlc && string.CompareOrdinal(entry.NodeId, local.Value.NodeId) > 0);
|
||||
|
||||
if (!remoteWins)
|
||||
{
|
||||
@@ -76,33 +90,28 @@ internal sealed class LwwApplier(SqliteLocalDb db, Func<DateTimeOffset>? utcNow
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!sqlByTable.TryGetValue(entry.TableName, out var sql))
|
||||
{
|
||||
sql = BuildTableSql(db.ReplicatedTables[entry.TableName]);
|
||||
sqlByTable[entry.TableName] = sql;
|
||||
}
|
||||
|
||||
// The row mutation and its ledger entry are TWO statements and must land together or
|
||||
// not at all: a failure between them commits a deleted/replaced row carrying a stale
|
||||
// ledger version, which SnapshotStreamer then warns about and skips — so that key never
|
||||
// travels again and the pair is silently, permanently divergent for it. A per-entry
|
||||
// savepoint makes the pair atomic without giving up the poison catch's per-ENTRY
|
||||
// granularity (the surrounding transaction, and every entry already applied in it,
|
||||
// survives a rollback to the savepoint). Cost measured on the path that matters — a
|
||||
// 100 000-row backlog drain — is ~7% (178.6k rows/s → 166.1k rows/s): the two extra
|
||||
// statements are prepared once like the rest and neither one touches the disk, since
|
||||
// an inner savepoint's release just merges into the enclosing transaction.
|
||||
var pairApplied = false;
|
||||
await sql.SavepointAsync(ct);
|
||||
try
|
||||
{
|
||||
var table = db.ReplicatedTables[entry.TableName];
|
||||
if (entry.IsTombstone)
|
||||
await tx.ExecuteAsync(sql.Delete, new { pk = entry.PkJson }, ct);
|
||||
await sql.DeleteAsync(table, entry.PkJson, ct);
|
||||
else
|
||||
await tx.ExecuteAsync(sql.Upsert, new { row = entry.RowJson }, ct);
|
||||
await sql.UpsertAsync(table, entry.RowJson!, ct);
|
||||
|
||||
await tx.ExecuteAsync(
|
||||
"INSERT OR REPLACE INTO __localdb_row_version " +
|
||||
"(table_name, pk_json, hlc, node_id, is_tombstone, tombstone_utc) " +
|
||||
"VALUES (@t, @pk, @hlc, @node, @tomb, @tut)",
|
||||
new
|
||||
{
|
||||
t = entry.TableName,
|
||||
pk = entry.PkJson,
|
||||
hlc = entry.Hlc,
|
||||
node = entry.NodeId,
|
||||
tomb = entry.IsTombstone ? 1 : 0,
|
||||
tut = entry.IsTombstone ? utcNow : null,
|
||||
}, ct);
|
||||
applied++;
|
||||
await sql.WriteRowVersionAsync(entry, utcNow, ct);
|
||||
pairApplied = true;
|
||||
}
|
||||
catch (SqliteException ex)
|
||||
{
|
||||
@@ -110,59 +119,224 @@ internal sealed class LwwApplier(SqliteLocalDb db, Func<DateTimeOffset>? utcNow
|
||||
// malformed row_json) are treated as per-entry poison — SQLite aborts just the
|
||||
// statement, not the transaction, so the batch continues after dead-lettering.
|
||||
// Any other exception propagates and rolls back the whole batch (fail-closed).
|
||||
await DeadLetterAsync(tx, entry, ex.Message, utcNow, ct);
|
||||
// The rollback comes FIRST so the dead-letter row it records is not discarded with
|
||||
// the half-applied pair.
|
||||
await sql.RollbackToSavepointAsync(ct);
|
||||
await sql.DeadLetterAsync(entry, ex.Message, utcNow, ct);
|
||||
deadLettered++;
|
||||
}
|
||||
|
||||
// Released outside the catch on purpose: a RELEASE that itself fails means the
|
||||
// transaction is no longer sound, and there is no savepoint state left to reason about,
|
||||
// so it propagates and rolls the whole batch back rather than being swallowed as poison.
|
||||
if (pairApplied)
|
||||
{
|
||||
await sql.ReleaseSavepointAsync(ct);
|
||||
applied++;
|
||||
}
|
||||
}
|
||||
|
||||
await tx.ExecuteAsync(
|
||||
"UPDATE __localdb_peer_state SET " +
|
||||
"last_applied_remote_seq = MAX(last_applied_remote_seq, @maxSeq), " +
|
||||
"last_seen_hlc = MAX(last_seen_hlc, @maxHlc), last_sync_utc = @utc WHERE id = 1",
|
||||
new { maxSeq, maxHlc, utc = utcNow }, ct);
|
||||
await sql.AdvancePeerStateAsync(maxSeq, maxHlc, utcNow, ct);
|
||||
await sql.SetApplyingAsync(0, ct);
|
||||
|
||||
LastBatchStatementCount = sql.StatementCount;
|
||||
|
||||
await tx.ExecuteAsync("UPDATE __localdb_applying SET applying = 0 WHERE id = 1", null, ct);
|
||||
await tx.CommitAsync(ct);
|
||||
|
||||
db.Clock.Observe(maxHlc);
|
||||
return new ApplyResult(maxSeq, applied, discarded, deadLettered);
|
||||
}
|
||||
|
||||
private static Task DeadLetterAsync(
|
||||
ILocalDbTransaction tx, OplogEntryRecord entry, string error, string utcNow, CancellationToken ct) =>
|
||||
tx.ExecuteAsync(
|
||||
"INSERT INTO __localdb_dead_letter " +
|
||||
"(received_utc, table_name, pk_json, row_json, hlc, node_id, error) " +
|
||||
"VALUES (@utc, @t, @pk, @row, @hlc, @node, @err)",
|
||||
new
|
||||
{
|
||||
utc = utcNow,
|
||||
t = entry.TableName,
|
||||
pk = entry.PkJson,
|
||||
row = entry.RowJson,
|
||||
hlc = entry.Hlc,
|
||||
node = entry.NodeId,
|
||||
err = error,
|
||||
}, ct);
|
||||
|
||||
private static (string Delete, string Upsert) BuildTableSql(ReplicatedTable table)
|
||||
/// <summary>
|
||||
/// The statement set one apply-batch runs on: each distinct statement is built and prepared on
|
||||
/// first use, then rebound and re-executed per row. Statements whose SQL depends on the table
|
||||
/// (delete / upsert) are cached per table; the rest are shared across all of them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Reusing the same <see cref="SqliteCommand"/> with unchanged <c>CommandText</c> is what makes
|
||||
/// this prepare-once: Microsoft.Data.Sqlite holds the compiled <c>sqlite3_stmt</c> on the command
|
||||
/// and only recompiles when the text changes. <see cref="SqliteCommand.Prepare"/> is called
|
||||
/// explicitly anyway, so the intent is stated rather than inferred from an implementation detail.
|
||||
/// </remarks>
|
||||
private sealed class PreparedStatements(SqliteConnection connection, SqliteTransaction transaction) : IDisposable
|
||||
{
|
||||
var name = Ident(table.Name);
|
||||
private readonly List<SqliteCommand> _owned = [];
|
||||
private readonly Dictionary<string, SqliteCommand> _deletes = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, SqliteCommand> _upserts = new(StringComparer.Ordinal);
|
||||
|
||||
var pkMatch = string.Join(" AND ", table.PkColumns.Select(p => $"{Ident(p)} = json_extract(@pk, {JsonPath(p)})"));
|
||||
var delete = $"DELETE FROM {name} WHERE {pkMatch}";
|
||||
private SqliteCommand? _applying;
|
||||
private SqliteCommand? _lookup;
|
||||
private SqliteCommand? _rowVersion;
|
||||
private SqliteCommand? _deadLetter;
|
||||
private SqliteCommand? _peerState;
|
||||
private SqliteCommand? _savepoint;
|
||||
private SqliteCommand? _releaseSavepoint;
|
||||
private SqliteCommand? _rollbackToSavepoint;
|
||||
|
||||
var columns = string.Join(", ", table.Columns.Select(c => Ident(c.Name)));
|
||||
var values = string.Join(", ", table.Columns.Select(c => $"json_extract(@row, {JsonPath(c.Name)})"));
|
||||
var upsert = $"INSERT OR REPLACE INTO {name} ({columns}) VALUES ({values})";
|
||||
/// <summary>The per-entry savepoint's name. Reused for every entry — one is open at a time.</summary>
|
||||
private const string SavepointName = "__localdb_apply_entry";
|
||||
|
||||
return (delete, upsert);
|
||||
/// <summary>How many distinct statements this set has prepared.</summary>
|
||||
public int StatementCount => _owned.Count;
|
||||
|
||||
/// <summary>Opens the per-entry savepoint that scopes one entry's row + ledger writes.</summary>
|
||||
public Task SavepointAsync(CancellationToken ct)
|
||||
{
|
||||
var cmd = _savepoint ??= Create("SAVEPOINT " + SavepointName);
|
||||
return cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
/// <summary>Commits the entry's savepoint into the surrounding batch transaction.</summary>
|
||||
public Task ReleaseSavepointAsync(CancellationToken ct)
|
||||
{
|
||||
var cmd = _releaseSavepoint ??= Create("RELEASE " + SavepointName);
|
||||
return cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Undoes everything the entry did and closes its savepoint. ROLLBACK TO leaves the savepoint
|
||||
/// OPEN, so the RELEASE is required — without it the names would nest one deeper per poison
|
||||
/// entry and the next rollback would unwind more than one entry's work.
|
||||
/// </summary>
|
||||
public async Task RollbackToSavepointAsync(CancellationToken ct)
|
||||
{
|
||||
var cmd = _rollbackToSavepoint ??= Create("ROLLBACK TO " + SavepointName);
|
||||
await cmd.ExecuteNonQueryAsync(ct);
|
||||
await ReleaseSavepointAsync(ct);
|
||||
}
|
||||
|
||||
public Task SetApplyingAsync(int value, CancellationToken ct)
|
||||
{
|
||||
var cmd = _applying ??= Create(
|
||||
"UPDATE __localdb_applying SET applying = @applying WHERE id = 1", "@applying");
|
||||
cmd.Parameters["@applying"].Value = value;
|
||||
return cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
public async Task<(long Hlc, string NodeId)?> ReadLocalVersionAsync(
|
||||
string tableName, string pkJson, CancellationToken ct)
|
||||
{
|
||||
var cmd = _lookup ??= Create(
|
||||
"SELECT hlc, node_id FROM __localdb_row_version WHERE table_name = @t AND pk_json = @pk",
|
||||
"@t", "@pk");
|
||||
cmd.Parameters["@t"].Value = tableName;
|
||||
cmd.Parameters["@pk"].Value = pkJson;
|
||||
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct);
|
||||
if (!await reader.ReadAsync(ct))
|
||||
return null;
|
||||
return (reader.GetInt64(0), reader.GetString(1));
|
||||
}
|
||||
|
||||
public Task DeleteAsync(ReplicatedTable table, string pkJson, CancellationToken ct)
|
||||
{
|
||||
if (!_deletes.TryGetValue(table.Name, out var cmd))
|
||||
_deletes[table.Name] = cmd = Create(BuildDelete(table), "@pk");
|
||||
cmd.Parameters["@pk"].Value = pkJson;
|
||||
return cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
public Task UpsertAsync(ReplicatedTable table, string rowJson, CancellationToken ct)
|
||||
{
|
||||
if (!_upserts.TryGetValue(table.Name, out var cmd))
|
||||
_upserts[table.Name] = cmd = Create(BuildUpsert(table), "@row");
|
||||
cmd.Parameters["@row"].Value = rowJson;
|
||||
return cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
public Task WriteRowVersionAsync(OplogEntryRecord entry, string utcNow, CancellationToken ct)
|
||||
{
|
||||
var cmd = _rowVersion ??= Create(
|
||||
"INSERT OR REPLACE INTO __localdb_row_version " +
|
||||
"(table_name, pk_json, hlc, node_id, is_tombstone, tombstone_utc) " +
|
||||
"VALUES (@t, @pk, @hlc, @node, @tomb, @tut)",
|
||||
"@t", "@pk", "@hlc", "@node", "@tomb", "@tut");
|
||||
cmd.Parameters["@t"].Value = entry.TableName;
|
||||
cmd.Parameters["@pk"].Value = entry.PkJson;
|
||||
cmd.Parameters["@hlc"].Value = entry.Hlc;
|
||||
cmd.Parameters["@node"].Value = entry.NodeId;
|
||||
cmd.Parameters["@tomb"].Value = entry.IsTombstone ? 1 : 0;
|
||||
cmd.Parameters["@tut"].Value = entry.IsTombstone ? utcNow : DBNull.Value;
|
||||
return cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
public Task DeadLetterAsync(OplogEntryRecord entry, string error, string utcNow, CancellationToken ct)
|
||||
{
|
||||
var cmd = _deadLetter ??= Create(
|
||||
"INSERT INTO __localdb_dead_letter " +
|
||||
"(received_utc, table_name, pk_json, row_json, hlc, node_id, error) " +
|
||||
"VALUES (@utc, @t, @pk, @row, @hlc, @node, @err)",
|
||||
"@utc", "@t", "@pk", "@row", "@hlc", "@node", "@err");
|
||||
cmd.Parameters["@utc"].Value = utcNow;
|
||||
cmd.Parameters["@t"].Value = entry.TableName;
|
||||
cmd.Parameters["@pk"].Value = entry.PkJson;
|
||||
cmd.Parameters["@row"].Value = (object?)entry.RowJson ?? DBNull.Value;
|
||||
cmd.Parameters["@hlc"].Value = entry.Hlc;
|
||||
cmd.Parameters["@node"].Value = entry.NodeId;
|
||||
cmd.Parameters["@err"].Value = error;
|
||||
return cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
public Task AdvancePeerStateAsync(long maxSeq, long maxHlc, string utcNow, CancellationToken ct)
|
||||
{
|
||||
var cmd = _peerState ??= Create(
|
||||
"UPDATE __localdb_peer_state SET " +
|
||||
"last_applied_remote_seq = MAX(last_applied_remote_seq, @maxSeq), " +
|
||||
"last_seen_hlc = MAX(last_seen_hlc, @maxHlc), last_sync_utc = @utc WHERE id = 1",
|
||||
"@maxSeq", "@maxHlc", "@utc");
|
||||
cmd.Parameters["@maxSeq"].Value = maxSeq;
|
||||
cmd.Parameters["@maxHlc"].Value = maxHlc;
|
||||
cmd.Parameters["@utc"].Value = utcNow;
|
||||
return cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
private SqliteCommand Create(string sql, params string[] parameterNames)
|
||||
{
|
||||
var cmd = connection.CreateCommand();
|
||||
try
|
||||
{
|
||||
cmd.Transaction = transaction;
|
||||
cmd.CommandText = sql;
|
||||
foreach (var name in parameterNames)
|
||||
cmd.Parameters.Add(new SqliteParameter(name, DBNull.Value));
|
||||
cmd.Prepare();
|
||||
_owned.Add(cmd);
|
||||
return cmd;
|
||||
}
|
||||
catch
|
||||
{
|
||||
cmd.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var cmd in _owned)
|
||||
cmd.Dispose();
|
||||
_owned.Clear();
|
||||
}
|
||||
|
||||
private static string BuildDelete(ReplicatedTable table)
|
||||
{
|
||||
var pkMatch = string.Join(
|
||||
" AND ", table.PkColumns.Select(p => $"{Ident(p)} = json_extract(@pk, {JsonPath(p)})"));
|
||||
return $"DELETE FROM {Ident(table.Name)} WHERE {pkMatch}";
|
||||
}
|
||||
|
||||
private static string BuildUpsert(ReplicatedTable table)
|
||||
{
|
||||
var columns = string.Join(", ", table.Columns.Select(c => Ident(c.Name)));
|
||||
var values = string.Join(", ", table.Columns.Select(c => $"json_extract(@row, {JsonPath(c.Name)})"));
|
||||
return $"INSERT OR REPLACE INTO {Ident(table.Name)} ({columns}) VALUES ({values})";
|
||||
}
|
||||
|
||||
// Local copy of TriggerSqlGenerator's convention (that helper lives in the core assembly).
|
||||
private static string Ident(string name) => "\"" + name.Replace("\"", "\"\"") + "\"";
|
||||
|
||||
// A single-quoted JSON path with a quoted key member, so a column name with reserved
|
||||
// characters still resolves: $."col". The key's double quotes are backslash-escaped per
|
||||
// JSON path grammar.
|
||||
private static string JsonPath(string column) => "'$.\"" + column.Replace("\"", "\\\"") + "\"'";
|
||||
}
|
||||
|
||||
// Local copy of TriggerSqlGenerator's convention (that helper lives in the core assembly).
|
||||
private static string Ident(string name) => "\"" + name.Replace("\"", "\"\"") + "\"";
|
||||
|
||||
// A single-quoted JSON path with a quoted key member, so a column name with reserved characters
|
||||
// still resolves: $."col". The key's double quotes are backslash-escaped per JSON path grammar.
|
||||
private static string JsonPath(string column) => "'$.\"" + column.Replace("\"", "\\\"") + "\"'";
|
||||
}
|
||||
|
||||
@@ -24,8 +24,9 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
|
||||
|
||||
/// <summary>
|
||||
/// Reads up to <paramref name="maxBatch"/> oplog entries with <c>seq > afterSeq</c> in ascending
|
||||
/// seq order. Caller contract: check <c>needs_snapshot</c> before trusting incremental reads —
|
||||
/// after cap-pruning, reads silently resume past a gap.
|
||||
/// seq order. Caller contract: this read resumes SILENTLY past a cap-pruned gap, so a caller
|
||||
/// streaming incrementally must first confirm its position is still below the horizon — see
|
||||
/// <see cref="GetMinSeqAsync"/>, which <c>SyncSession</c>'s pump checks on every iteration.
|
||||
/// </summary>
|
||||
public Task<IReadOnlyList<OplogEntryRecord>> ReadBatchAboveAsync(long afterSeq, int maxBatch, CancellationToken ct = default) =>
|
||||
db.QueryAsync(
|
||||
@@ -70,12 +71,42 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
|
||||
"DELETE FROM __localdb_row_version WHERE is_tombstone = 1 AND tombstone_utc < @cutoff",
|
||||
new { cutoff = FormatUtc(_utcNow() - options.TombstoneRetention) }, ct);
|
||||
|
||||
/// <summary>Counts the unacked backlog: oplog entries above the peer's last-acked watermark.</summary>
|
||||
/// <summary>
|
||||
/// The unacked backlog — oplog entries above the peer's last-acked watermark — in O(1).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why not <c>COUNT(*)</c>.</b> The obvious query walks every unacked row, and this value is
|
||||
/// read once a minute by maintenance AND on every metrics scrape. A node whose peer has been
|
||||
/// down for a day therefore pays a full scan of a million-row backlog, repeatedly, precisely
|
||||
/// when it is already under stress.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The formula.</b> <c>MAX(seq) - MAX(last_acked_seq, MIN(seq) - 1)</c>, floored at 0. Both
|
||||
/// MIN and MAX over <c>seq</c> are single b-tree seeks (it is the rowid, and SQLite's min/max
|
||||
/// optimization turns each into one seek), and the peer-state read is a one-row lookup — so the
|
||||
/// whole thing is three seeks regardless of backlog size.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why it is exact.</b> <c>seq</c> is AUTOINCREMENT, so retained rows are a contiguous run
|
||||
/// wherever nothing has punched a hole in the middle — and no ordinary path punches holes: ack
|
||||
/// pruning deletes <c>seq <= last_acked_seq</c> (a prefix), the row cap deletes below
|
||||
/// <c>MAX(seq) - keep + 1</c> (a prefix), and the age cap deletes below an HLC cutoff, which is
|
||||
/// a prefix because HLCs are stamped in seq order. Clamping the floor to <c>MIN(seq) - 1</c> is
|
||||
/// what keeps the answer exact after a prune has moved the horizon above the ack watermark;
|
||||
/// without it a cap-pruned oplog would report its whole seq range forever and re-flag the cap on
|
||||
/// every tick.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The one inexact case</b> is <c>ILocalDb.DeregisterReplicated</c>, which deletes one
|
||||
/// table's rows from wherever they sit and so can leave holes. The estimate then over-reports by
|
||||
/// at most the number of holes, until the watermark advances past them. Over-reporting is the
|
||||
/// safe direction: it can only trigger a snapshot resync earlier, never suppress one.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public async Task<long> GetOplogDepthAsync(CancellationToken ct = default)
|
||||
{
|
||||
var rows = await db.QueryAsync(
|
||||
"SELECT COUNT(*) FROM __localdb_oplog WHERE seq > (SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1)",
|
||||
static r => r.GetInt64(0), null, ct);
|
||||
var rows = await db.QueryAsync(DepthSql, static r => r.GetInt64(0), null, ct);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
@@ -84,7 +115,7 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
|
||||
/// metric-observation callbacks (gauge scrape / status poll); mirrors
|
||||
/// <see cref="GetOplogDepthAsync"/>'s query. Deliberate tradeoff: synchronous I/O on the
|
||||
/// metrics-collection thread is acceptable here because WAL-mode readers never block (or are
|
||||
/// blocked by) writers, the COUNT is indexed-cheap, and the worst case is bounded by
|
||||
/// blocked by) writers, the query is three b-tree seeks, and the worst case is bounded by
|
||||
/// <c>BusyTimeoutMs</c> — all at scrape frequency, not on any data path. May throw if the db
|
||||
/// is disposing — callers on the observation path swallow and drop the sample.
|
||||
/// </summary>
|
||||
@@ -92,11 +123,18 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
|
||||
{
|
||||
using var conn = db.CreateConnection();
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText =
|
||||
"SELECT COUNT(*) FROM __localdb_oplog WHERE seq > (SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1)";
|
||||
cmd.CommandText = DepthSql;
|
||||
return (long)cmd.ExecuteScalar()!;
|
||||
}
|
||||
|
||||
// Both depth accessors run exactly this; keeping it in one place is what makes "mirrors the
|
||||
// async query" a fact rather than a comment that can rot.
|
||||
internal const string DepthSql =
|
||||
"SELECT MAX(0, " +
|
||||
"COALESCE((SELECT MAX(seq) FROM __localdb_oplog), 0) - " +
|
||||
"MAX((SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1), " +
|
||||
"COALESCE((SELECT MIN(seq) FROM __localdb_oplog), 1) - 1))";
|
||||
|
||||
/// <summary>
|
||||
/// The highest seq this node has ever produced: the oplog's max, or — once acked rows have been
|
||||
/// pruned away — the ack watermark that pruned them. Nothing we emit can exceed it, so a peer
|
||||
@@ -111,6 +149,24 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The oldest oplog seq still present — the delta horizon — or 0 when the oplog is empty.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// One b-tree seek (<c>seq</c> is the rowid and SQLite optimizes MIN into a seek), which is what
|
||||
/// makes it affordable for the pump to check on every flush interval. Compared against a
|
||||
/// session's sent watermark it answers the only question the pump has: is everything I have not
|
||||
/// yet sent still here? Ack-pruning deletes <c>seq <= last_acked_seq</c> and the peer can only
|
||||
/// ack what it received, so in a healthy session the horizon never climbs past
|
||||
/// <c>sentThruSeq + 1</c>; a higher one means a cap prune ate unsent deltas.
|
||||
/// </remarks>
|
||||
public async Task<long> GetMinSeqAsync(CancellationToken ct = default)
|
||||
{
|
||||
var rows = await db.QueryAsync(
|
||||
"SELECT COALESCE(MIN(seq), 0) FROM __localdb_oplog", static r => r.GetInt64(0), null, ct);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
/// <summary>The highest oplog seq present, or 0 when the oplog is empty.</summary>
|
||||
public async Task<long> GetMaxSeqAsync(CancellationToken ct = default)
|
||||
{
|
||||
@@ -133,9 +189,9 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
|
||||
// Condition reads and prunes share the transaction so the decision cannot go stale (TOCTOU).
|
||||
await using var tx = await db.BeginTransactionAsync(ct);
|
||||
|
||||
var depth = (await tx.QueryAsync(
|
||||
"SELECT COUNT(*) FROM __localdb_oplog WHERE seq > (SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1)",
|
||||
static r => r.GetInt64(0), null, ct))[0];
|
||||
// Same O(1) expression as GetOplogDepthAsync — the caps check is the per-minute reader that
|
||||
// made the old COUNT(*) expensive in the first place.
|
||||
var depth = (await tx.QueryAsync(DepthSql, static r => r.GetInt64(0), null, ct))[0];
|
||||
var overRows = depth > options.MaxOplogRows;
|
||||
|
||||
var oldest = await tx.QueryAsync(
|
||||
@@ -196,10 +252,61 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
|
||||
db.ExecuteAsync(
|
||||
"UPDATE __localdb_peer_state SET last_applied_remote_seq = 0, last_seen_hlc = 0 WHERE id = 1", null, ct);
|
||||
|
||||
/// <summary>Sets or clears the snapshot-resync-required flag.</summary>
|
||||
/// <summary>
|
||||
/// Sets or clears the DATABASE-WIDE snapshot-resync flag — the one that means "every registered
|
||||
/// table". Set by the oplog-horizon paths (cap prune, impossible peer ack), which invalidate
|
||||
/// delta continuity for all tables at once. Per-table flags live in
|
||||
/// <c>__localdb_snapshot_state</c>; see <see cref="GetTablesNeedingSnapshotAsync"/>.
|
||||
/// </summary>
|
||||
public Task SetNeedsSnapshotAsync(bool value, CancellationToken ct = default) =>
|
||||
db.ExecuteAsync("UPDATE __localdb_peer_state SET needs_snapshot = @v WHERE id = 1", new { v = value ? 1 : 0 }, ct);
|
||||
|
||||
/// <summary>
|
||||
/// The tables individually flagged as owing the peer a snapshot (schema v3), in ordinal name
|
||||
/// order. Independent of the database-wide flag: baselining one table flags only that table.
|
||||
/// </summary>
|
||||
public Task<IReadOnlyList<string>> GetTablesNeedingSnapshotAsync(CancellationToken ct = default) =>
|
||||
db.QueryAsync(
|
||||
"SELECT table_name FROM __localdb_snapshot_state WHERE needs_snapshot <> 0 ORDER BY table_name",
|
||||
static r => r.GetString(0), null, ct);
|
||||
|
||||
/// <summary>
|
||||
/// Clears the snapshot debt a completed snapshot discharged: the named tables when
|
||||
/// <paramref name="tables"/> is non-empty, otherwise <b>everything</b> — the database-wide flag
|
||||
/// and every per-table row, because a full snapshot subsumes each of them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Rows are deleted rather than zeroed: absence is the resting state, and keeping a row per
|
||||
/// table that has ever been baselined would accumulate bookkeeping that means nothing. Runs in
|
||||
/// one transaction so a full snapshot's two clears cannot half-apply and leave the pair
|
||||
/// re-snapshotting on every reconnect.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A scoped clear deliberately does NOT touch the database-wide flag. If the cap enforcer
|
||||
/// flagged a whole-database resync while this snapshot was in flight, that debt is real and
|
||||
/// unrelated — the snapshot only carried some of the tables — so it must survive. The full
|
||||
/// clear may take it down because a full snapshot carries every row state as of its as-of seq,
|
||||
/// which is exactly what the pruned deltas would have delivered.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public async Task ClearSnapshotDebtAsync(IReadOnlyList<string> tables, CancellationToken ct = default)
|
||||
{
|
||||
await using var tx = await db.BeginTransactionAsync(ct);
|
||||
if (tables.Count == 0)
|
||||
{
|
||||
await tx.ExecuteAsync("UPDATE __localdb_peer_state SET needs_snapshot = 0 WHERE id = 1", null, ct);
|
||||
await tx.ExecuteAsync("DELETE FROM __localdb_snapshot_state", null, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (var table in tables)
|
||||
await tx.ExecuteAsync(
|
||||
"DELETE FROM __localdb_snapshot_state WHERE table_name = @t", new { t = table }, ct);
|
||||
}
|
||||
await tx.CommitAsync(ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records the highest remote seq applied locally plus the peer HLC observed with it; also stamps
|
||||
/// <c>last_sync_utc</c>. Both watermarks are MAX-guarded (monotonic): a snapshot completing with a
|
||||
|
||||
@@ -19,21 +19,57 @@ internal interface ISnapshotApplier
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Streams a full snapshot of every registered table to a peer that has fallen behind the pruned
|
||||
/// oplog horizon. Reads the authoritative per-row version from <c>__localdb_row_version</c> joined
|
||||
/// Which tables a snapshot covers. An EMPTY <see cref="Tables"/> is the wire's own encoding of
|
||||
/// "all registered tables" (see <c>SnapshotBegin.tables</c>) and is what every pre-0.3.0 peer both
|
||||
/// sends and expects, so it is deliberately the same value here rather than a separate flag.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The distinction is not cosmetic: a FULL snapshot carries every row state as of its
|
||||
/// <c>as_of_seq</c>, so the receiver may jump its watermark there and the sender may skip the
|
||||
/// deltas it covered. A PARTIAL one carries only the listed tables, so neither side may move a
|
||||
/// watermark that also speaks for the tables it left out.
|
||||
/// </remarks>
|
||||
internal sealed record SnapshotScope(IReadOnlyList<string> Tables)
|
||||
{
|
||||
/// <summary>Every registered table — the only scope a pre-0.3.0 peer can be sent.</summary>
|
||||
public static readonly SnapshotScope Full = new([]);
|
||||
|
||||
public bool IsPartial => Tables.Count > 0;
|
||||
|
||||
public bool Includes(string tableName) =>
|
||||
Tables.Count == 0 || Tables.Contains(tableName, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Streams a snapshot of the tables in the requested <see cref="SnapshotScope"/> — every registered
|
||||
/// table for a peer that has fallen behind the pruned oplog horizon, or just the baselined ones for
|
||||
/// a scoped resync. Reads the authoritative per-row version from <c>__localdb_row_version</c> joined
|
||||
/// to each live table, pages it into <see cref="SnapshotBatch"/> messages through the gated data
|
||||
/// lane, and brackets them with SnapshotBegin/SnapshotComplete carrying the max oplog seq as-of.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Clearing the snapshot debt is deliberately NOT done here: whether the flags may be cleared at all
|
||||
/// depends on the peer's capability (a 0.3.0 peer acknowledges the snapshot, a pre-0.3.0 one cannot),
|
||||
/// which is session state. <see cref="SyncSession"/> owns that decision; this type only streams.
|
||||
/// </remarks>
|
||||
internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, ReplicationOptions options, ILogger logger)
|
||||
{
|
||||
/// <summary>Streams the full snapshot; returns the as-of oplog seq it covers, so the session can skip already-covered tail deltas.</summary>
|
||||
public async Task<long> SendAsync(Func<SyncMessage, CancellationToken, Task> send, CancellationToken ct)
|
||||
/// <summary>
|
||||
/// Streams the snapshot for <paramref name="scope"/>; returns the as-of oplog seq it covers, so
|
||||
/// the session can skip already-covered tail deltas (a FULL snapshot only — a partial one speaks
|
||||
/// for nothing outside its tables).
|
||||
/// </summary>
|
||||
public async Task<long> SendAsync(SnapshotScope scope, Func<SyncMessage, CancellationToken, Task> send, CancellationToken ct)
|
||||
{
|
||||
var asOfSeq = await store.GetMaxSeqAsync(ct);
|
||||
await send(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = asOfSeq } }, ct);
|
||||
var begin = new SnapshotBegin { AsOfSeq = asOfSeq };
|
||||
begin.Tables.AddRange(scope.Tables);
|
||||
await send(new SyncMessage { SnapshotBegin = begin }, ct);
|
||||
|
||||
var totalRows = 0L;
|
||||
foreach (var table in db.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal))
|
||||
foreach (var table in db.ReplicatedTables.Values
|
||||
.Where(t => scope.Includes(t.Name))
|
||||
.OrderBy(t => t.Name, StringComparer.Ordinal))
|
||||
{
|
||||
var sql = BuildPageSql(table);
|
||||
// Keyset cursor (pk_json > last seen), not LIMIT/OFFSET: concurrent inserts/prunes
|
||||
@@ -57,7 +93,13 @@ internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, Repli
|
||||
break;
|
||||
lastPk = rows[^1].PkJson;
|
||||
|
||||
// One DB page may span several messages: the page is bounded by ROW COUNT, but a
|
||||
// message is bounded by BYTES (ReplicationOptions.MaxBatchBytes) — the same budget
|
||||
// the delta pump honours, and for the same reason. Wide rows are exactly why a
|
||||
// snapshot is the easiest way to exceed the transport's message limit: it ships
|
||||
// whole rows, all of them, back to back.
|
||||
var batch = new SnapshotBatch { TableName = table.Name };
|
||||
var bytes = 0L;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (!row.IsTombstone && row.RowJson is null)
|
||||
@@ -79,11 +121,25 @@ internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, Repli
|
||||
};
|
||||
if (row.RowJson is not null)
|
||||
snapshotRow.RowJson = row.RowJson;
|
||||
|
||||
var size = RowSize(snapshotRow);
|
||||
// Flush before adding, never after: a message keeps at least one row however
|
||||
// large, so an oversized row goes out alone rather than stalling the snapshot.
|
||||
if (batch.Rows.Count > 0 && bytes + size > options.MaxBatchBytes)
|
||||
{
|
||||
await send(new SyncMessage { SnapshotBatch = batch }, ct);
|
||||
totalRows += batch.Rows.Count;
|
||||
batch = new SnapshotBatch { TableName = table.Name };
|
||||
bytes = 0;
|
||||
}
|
||||
batch.Rows.Add(snapshotRow);
|
||||
bytes += size;
|
||||
}
|
||||
if (batch.Rows.Count > 0)
|
||||
{
|
||||
await send(new SyncMessage { SnapshotBatch = batch }, ct);
|
||||
totalRows += batch.Rows.Count;
|
||||
totalRows += batch.Rows.Count;
|
||||
}
|
||||
|
||||
if (rows.Count < options.MaxBatchSize)
|
||||
break;
|
||||
@@ -91,24 +147,29 @@ internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, Repli
|
||||
}
|
||||
|
||||
await send(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = asOfSeq } }, ct);
|
||||
// The sender owed this snapshot (its own needs_snapshot); clearing it here — after the whole
|
||||
// snapshot is on the wire — stops it re-streaming on the next reconnect. Narrow window: the
|
||||
// messages are enqueued on the data lane, not yet transport-delivered, so a crash/drop after
|
||||
// this clear could lose the snapshot with the flag already down. The reconnect handshake's
|
||||
// gap check (peer watermark below the pruned horizon) re-detects that in every case where the
|
||||
// oplog is non-empty; the sole undetectable corner is an EMPTY oplog (nothing to gap-check),
|
||||
// where the peer stays behind until new local writes recreate a horizon. Accepted: closing it
|
||||
// would need transport-level delivery acks for snapshot frames.
|
||||
await store.SetNeedsSnapshotAsync(false, ct);
|
||||
logger.LogInformation("Snapshot sent (as_of_seq {AsOfSeq}, {Rows} rows).", asOfSeq, totalRows);
|
||||
logger.LogInformation(
|
||||
"Snapshot sent (as_of_seq {AsOfSeq}, {Rows} rows, scope {Scope}).",
|
||||
asOfSeq, totalRows, scope.IsPartial ? string.Join(",", scope.Tables) : "all");
|
||||
return asOfSeq;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Serialized bytes one row contributes to its batch message: the row's own encoding plus the
|
||||
/// repeated field's tag and length prefix, as protobuf itself computes them.
|
||||
/// </summary>
|
||||
private static long RowSize(SnapshotRow row) =>
|
||||
1 + Google.Protobuf.CodedOutputStream.ComputeMessageSize(row);
|
||||
|
||||
// Pages one table's rows by joining the row-version ledger to the live table on the PK columns
|
||||
// extracted from pk_json. Rows present in the live table but absent from __localdb_row_version
|
||||
// (data written BEFORE the table was registered for replication) are intentionally NOT snapshotted:
|
||||
// they were never captured, consistent with CDC semantics — a full re-registration is required to
|
||||
// baseline them.
|
||||
// (data written BEFORE the table was registered for replication) are NOT snapshotted: they were
|
||||
// never captured, consistent with CDC semantics.
|
||||
//
|
||||
// That is a real hole for a node that starts replicating LATE — its whole pre-existing dataset is
|
||||
// invisible to the peer, silently and permanently — and the supported way to close it is to
|
||||
// register with `ILocalDb.RegisterReplicated(table, baselineExistingRows: true)`, which seeds the
|
||||
// ledger at an LWW-safe floor and flags the resync that carries the rows here. Nothing changes on
|
||||
// this side: baselined rows are ordinary ledger rows and page like any other.
|
||||
private static string BuildPageSql(ReplicatedTable table)
|
||||
{
|
||||
var name = Ident(table.Name);
|
||||
@@ -149,6 +210,7 @@ internal sealed class SnapshotApplier : ISnapshotApplier
|
||||
private long _asOfSeq;
|
||||
private long _rows;
|
||||
private long _maxHlc;
|
||||
private bool _partial;
|
||||
|
||||
// db is part of the collaborator's canonical shape (mirrors the streamer) even though the merge
|
||||
// reaches the database through the applier and store; kept for construction symmetry in the factory.
|
||||
@@ -165,7 +227,13 @@ internal sealed class SnapshotApplier : ISnapshotApplier
|
||||
_asOfSeq = begin.AsOfSeq;
|
||||
_rows = 0;
|
||||
_maxHlc = 0;
|
||||
_logger.LogInformation("Snapshot receive begun (as_of_seq {AsOfSeq}).", _asOfSeq);
|
||||
// A non-empty table list scopes the snapshot; empty (which is all a pre-0.3.0 sender can
|
||||
// emit) means every registered table. Read here rather than on complete because
|
||||
// SnapshotComplete does not repeat it — the receive loop guarantees a Begin precedes both.
|
||||
_partial = begin.Tables.Count > 0;
|
||||
_logger.LogInformation(
|
||||
"Snapshot receive begun (as_of_seq {AsOfSeq}, scope {Scope}).",
|
||||
_asOfSeq, _partial ? string.Join(",", begin.Tables) : "all");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
@@ -187,9 +255,20 @@ internal sealed class SnapshotApplier : ISnapshotApplier
|
||||
|
||||
public async Task OnCompleteAsync(SnapshotComplete complete, CancellationToken ct)
|
||||
{
|
||||
// A PARTIAL snapshot may not move the inbound seq watermark. That watermark is one number
|
||||
// for the whole peer oplog, and jumping it to as_of_seq would declare every table applied
|
||||
// through that point — including the ones this snapshot deliberately left out, whose deltas
|
||||
// below as_of_seq the sender would then never resend. Only the HLC is carried across (seq 0
|
||||
// is a no-op against the MAX guard), so the clock still cannot regress below what the peer
|
||||
// has stamped. The sender makes the mirror-image concession: it does not advance its own
|
||||
// sent-thru watermark past a partial snapshot, so those deltas are still pumped normally
|
||||
// and merely re-deliver rows the snapshot already carried, which LWW absorbs.
|
||||
var seq = _partial ? 0 : complete.AsOfSeq;
|
||||
// SetLastAppliedRemoteSeqAsync is MAX-guarded, so a delta applied mid-snapshot with a seq
|
||||
// above the snapshot's as-of (or an HLC above what the snapshot carried) is never regressed.
|
||||
await _store.SetLastAppliedRemoteSeqAsync(complete.AsOfSeq, _maxHlc, ct);
|
||||
_logger.LogInformation("Snapshot receive complete (as_of_seq {AsOfSeq}, {Rows} rows).", complete.AsOfSeq, _rows);
|
||||
await _store.SetLastAppliedRemoteSeqAsync(seq, _maxHlc, ct);
|
||||
_logger.LogInformation(
|
||||
"Snapshot receive complete (as_of_seq {AsOfSeq}, {Rows} rows, partial {Partial}).",
|
||||
complete.AsOfSeq, _rows, _partial);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,11 +10,13 @@ namespace ZB.MOM.WW.LocalDb.Replication.Internal;
|
||||
/// <summary>
|
||||
/// The symmetric 2-node sync protocol as one state machine: both the gRPC service and the gRPC
|
||||
/// client run this identical logic over a <see cref="SyncDuplex"/>. It performs the fail-closed
|
||||
/// handshake, then runs a bidirectional steady state — an outbound change pump and an inbound
|
||||
/// apply/ack loop — until the token is cancelled or the stream ends. All outbound traffic flows
|
||||
/// through a single writer task fed by two lanes: an unbounded CONTROL lane (handshake messages,
|
||||
/// acks) and a bounded DATA lane (delta/snapshot batches). Receiving therefore never blocks on the
|
||||
/// ability to write — acks always fit — while the bounded data lane is the pump's backpressure.
|
||||
/// handshake, then runs a bidirectional steady state until the token is cancelled or the stream
|
||||
/// ends: an outbound phase (the full snapshot this node owes, if any, then the change pump) and an
|
||||
/// inbound apply/ack loop, STARTED TOGETHER so the inbox keeps draining even while a snapshot is
|
||||
/// being sent. All outbound traffic flows through a single writer task fed by two lanes: an
|
||||
/// unbounded CONTROL lane (handshake messages, acks) and a bounded DATA lane (delta/snapshot
|
||||
/// batches). Receiving therefore never blocks on the ability to write — acks always fit — while the
|
||||
/// bounded data lane is the pump's backpressure.
|
||||
/// Local writes never block on the peer; the first fault tears down all loops and propagates.
|
||||
/// </summary>
|
||||
internal sealed class SyncSession
|
||||
@@ -25,6 +27,10 @@ internal sealed class SyncSession
|
||||
// inbound batch rate (the peer's flow control), so its lane can safely be unbounded.
|
||||
private const int DataLaneCapacity = 16;
|
||||
|
||||
// How long a faulting session's teardown lets the writer keep flushing already-enqueued
|
||||
// messages before the backstop cancel; only ever reached when the transport itself stalls.
|
||||
private static readonly TimeSpan FaultFlushGrace = TimeSpan.FromSeconds(2);
|
||||
|
||||
private readonly SqliteLocalDb _db;
|
||||
private readonly OplogStore _store;
|
||||
private readonly LwwApplier _applier;
|
||||
@@ -45,6 +51,17 @@ internal sealed class SyncSession
|
||||
// single receive-loop task, so no synchronization is needed.
|
||||
private bool _snapshotReceiving;
|
||||
|
||||
// Whether the peer speaks the 0.3.0 snapshot extensions: scoped snapshots
|
||||
// (SnapshotBegin.tables) and the SnapshotAck that answers a completed one. Written once during
|
||||
// the handshake — which completes before either loop starts — then read-only.
|
||||
private bool _peerSupportsPartialSnapshot;
|
||||
|
||||
// The scope of the snapshot this node is streaming, held until the peer's SnapshotAck discharges
|
||||
// it; null on every session that owes none. Assigned on the pump task strictly BEFORE the
|
||||
// snapshot's first message is enqueued and read on the receive loop when the ack lands, so the
|
||||
// ordering is causal — volatile only to make the cross-thread publication explicit.
|
||||
private SnapshotScope? _sentSnapshotScope;
|
||||
|
||||
public SyncSession(
|
||||
SqliteLocalDb db, OplogStore store, LwwApplier applier, ReplicationOptions options, ILogger logger,
|
||||
Func<DateTimeOffset>? utcNow = null, LocalDbMetrics? metrics = null, SyncStatus? status = null)
|
||||
@@ -61,11 +78,12 @@ internal sealed class SyncSession
|
||||
|
||||
/// <summary>
|
||||
/// Invoked when THIS node owes the peer a snapshot (its own snapshot-required computation, not
|
||||
/// the peer's) before deltas resume. Receives a gated send delegate that enqueues on the bounded
|
||||
/// data lane (never the raw transport); returns the as-of oplog seq the snapshot covers so the
|
||||
/// pump can skip already-covered tail deltas. Wired to <see cref="SnapshotStreamer.SendAsync"/>.
|
||||
/// the peer's) before deltas resume. Receives the negotiated <see cref="SnapshotScope"/> and a
|
||||
/// gated send delegate that enqueues on the bounded data lane (never the raw transport); returns
|
||||
/// the as-of oplog seq the snapshot covers so the pump can skip already-covered tail deltas.
|
||||
/// Wired to <see cref="SnapshotStreamer.SendAsync"/>.
|
||||
/// </summary>
|
||||
public Func<Func<SyncMessage, CancellationToken, Task>, CancellationToken, Task<long>>? SnapshotSender { get; set; }
|
||||
public Func<SnapshotScope, Func<SyncMessage, CancellationToken, Task>, CancellationToken, Task<long>>? SnapshotSender { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Consumes an inbound snapshot inline on the receive loop: SnapshotBegin opens the receive,
|
||||
@@ -97,10 +115,19 @@ internal sealed class SyncSession
|
||||
var writer = WriterLoopAsync(duplex, control.Reader, data.Reader, linked.Token);
|
||||
try
|
||||
{
|
||||
await HandshakeAsync(duplex, control.Writer, GatedDataSend, linked.Token);
|
||||
var snapshotScope = await HandshakeAsync(duplex, control.Writer, linked.Token);
|
||||
|
||||
var pump = PumpLoopAsync(data.Writer, linked.Token);
|
||||
// The receive loop starts BEFORE any snapshot this node owes is streamed. That ordering
|
||||
// is load-bearing, not incidental: a snapshot sent from inside the handshake left the
|
||||
// inbox undrained for its whole duration, so a MUTUAL snapshot (both sides flagged —
|
||||
// e.g. both baselined an existing dataset) required an unbounded inbox to avoid
|
||||
// deadlock, and therefore buffered a whole peer dataset in memory. With the receive loop
|
||||
// running throughout, the peer's snapshot is applied as it arrives, the inbox can be
|
||||
// bounded (see GrpcSyncDuplex.InboxCapacity), and mutual snapshots exchange under
|
||||
// ordinary flow control. Interleaving is already safe by design: snapshot rows and
|
||||
// deltas both merge through LWW, and the snapshot's watermark write is MAX-guarded.
|
||||
var receive = ReceiveLoopAsync(duplex, control.Writer, linked.Token);
|
||||
var pump = SnapshotThenPumpLoopAsync(snapshotScope, GatedDataSend, data.Writer, linked.Token);
|
||||
var loops = new[] { pump, receive, writer };
|
||||
|
||||
// The first COMPLETED task is the primary outcome. Cancel the siblings, observe every
|
||||
@@ -117,15 +144,33 @@ internal sealed class SyncSession
|
||||
{
|
||||
// Handshake-phase faults (and the primary rethrow above) land here: the writer may
|
||||
// still be running, so stop and observe it before the primary exception surfaces.
|
||||
linked.Cancel();
|
||||
// It must be allowed to DRAIN, not be cancelled outright: the Handshake enqueued at
|
||||
// the top of HandshakeAsync can still be sitting in the control lane (the enqueue →
|
||||
// receive → validate → fault path has no yields, while the writer's wakeup waits in
|
||||
// the thread-pool queue), and that message is the only thing that lets the peer
|
||||
// compute its own fail-closed verdict — mismatch detection is symmetric, so this
|
||||
// node's locally-thrown verdict reaches the peer only as the Handshake it transmits.
|
||||
// Cancelling first killed it almost every time the peer's handshake was already
|
||||
// buffered, leaving the healthy peer to misread a schema mismatch as a generic
|
||||
// disconnect and reconnect-loop at the fast backoff. Completing the lanes lets the
|
||||
// writer send what is already enqueued and exit on its own; the deferred cancel is
|
||||
// the backstop that keeps a wedged transport from holding teardown hostage.
|
||||
control.Writer.TryComplete();
|
||||
data.Writer.TryComplete();
|
||||
linked.CancelAfter(FaultFlushGrace);
|
||||
await DrainSecondaryAsync(writer);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandshakeAsync(
|
||||
SyncDuplex duplex, ChannelWriter<SyncMessage> controlLane,
|
||||
Func<SyncMessage, CancellationToken, Task> dataSend, CancellationToken ct)
|
||||
/// <summary>
|
||||
/// Runs the two-phase fail-closed handshake and returns the snapshot THIS node owes the peer —
|
||||
/// null for none, otherwise the <see cref="SnapshotScope"/> to stream (its own computation from
|
||||
/// its own state; the peer's answer drives the peer's send direction, symmetrically). Sending
|
||||
/// that snapshot is deliberately NOT part of the handshake; see <see cref="RunAsync"/>.
|
||||
/// </summary>
|
||||
private async Task<SnapshotScope?> HandshakeAsync(
|
||||
SyncDuplex duplex, ChannelWriter<SyncMessage> controlLane, CancellationToken ct)
|
||||
{
|
||||
var peerState = await _store.GetPeerStateAsync(ct);
|
||||
var localHandshake = BuildHandshake(peerState);
|
||||
@@ -142,6 +187,11 @@ internal sealed class SyncSession
|
||||
|
||||
ValidateHandshake(localHandshake, peerHandshake);
|
||||
|
||||
// Read before anything can act on it. A pre-0.3.0 peer leaves the field at its proto3
|
||||
// default, so "false" covers both "explicitly unsupported" and "older build" without a
|
||||
// version check — which is the point of advertising a capability rather than a version.
|
||||
_peerSupportsPartialSnapshot = peerHandshake.SupportsPartialSnapshot;
|
||||
|
||||
// A different node id on the far end means the peer was replaced, not merely restarted — a
|
||||
// rebuilt host, a restored-from-nothing volume. Its oplog numbers from 1 again, so the
|
||||
// watermark we hold (in the OLD peer's seq space) would silently swallow its first N
|
||||
@@ -175,34 +225,78 @@ internal sealed class SyncSession
|
||||
_sentThruSeq = peerAppliedThruSeq;
|
||||
Interlocked.Exchange(ref _peerAckedSeq, peerAppliedThruSeq);
|
||||
|
||||
var snapshotRequired = await ComputeSnapshotRequiredAsync(peerHandshake, peerState, ct);
|
||||
var snapshotScope = await ComputeSnapshotScopeAsync(peerHandshake, peerState, ct);
|
||||
await controlLane.WriteAsync(
|
||||
new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = _db.NodeId, SnapshotRequired = snapshotRequired } }, ct);
|
||||
new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = _db.NodeId, SnapshotRequired = snapshotScope is not null } }, ct);
|
||||
|
||||
var peerAckMsg = await ReceiveOneAsync(duplex, ct);
|
||||
if (peerAckMsg.MsgCase != SyncMessage.MsgOneofCase.HandshakeAck)
|
||||
throw new InvalidOperationException(
|
||||
$"Replication protocol error: expected a HandshakeAck, got {peerAckMsg.MsgCase}.");
|
||||
|
||||
// We stream the snapshot when WE owe the peer one (computed from our pruned horizon /
|
||||
// needs_snapshot), then it lands on the peer's receive loop. The peer's own flag drives ITS
|
||||
// send direction symmetrically; both are safe to run because LWW merges either way.
|
||||
// Memory profile of a MUTUAL snapshot: while we send here (receive loop not yet running),
|
||||
// the peer's inbound frames — potentially its entire snapshot — buffer in the unbounded
|
||||
// duplex inbox until our send completes. Worst case one full peer snapshot held in memory;
|
||||
// acceptable for the small LocalDb datasets this library targets.
|
||||
if (snapshotRequired)
|
||||
return snapshotScope;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The outbound steady state: stream the full snapshot this node owes the peer (if any), then
|
||||
/// run the incremental delta pump. One task, because the ordering between them is a protocol
|
||||
/// requirement — the pump must not put deltas on the data lane ahead of the snapshot, and it
|
||||
/// starts from the as-of seq the snapshot covered.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Split out of the handshake so the receive loop is already draining the inbox while this
|
||||
/// streams (see <see cref="RunAsync"/>). Nothing else about the snapshot changes: it goes
|
||||
/// through the same gated data lane, so it is still bounded by the writer's flow control rather
|
||||
/// than racing the transport.
|
||||
/// </remarks>
|
||||
private async Task SnapshotThenPumpLoopAsync(
|
||||
SnapshotScope? snapshotScope, Func<SyncMessage, CancellationToken, Task> dataSend,
|
||||
ChannelWriter<SyncMessage> dataLane, CancellationToken ct)
|
||||
{
|
||||
if (snapshotScope is not null)
|
||||
{
|
||||
if (SnapshotSender is null)
|
||||
throw new NotSupportedException("snapshot required but no snapshot sender configured");
|
||||
var snapshotAsOfSeq = await SnapshotSender(dataSend, ct);
|
||||
// Published BEFORE the first frame is enqueued, so the peer's SnapshotAck can never
|
||||
// arrive on the receive loop while this is still null.
|
||||
Volatile.Write(ref _sentSnapshotScope, snapshotScope);
|
||||
var snapshotAsOfSeq = await SnapshotSender(snapshotScope, dataSend, ct);
|
||||
_metrics?.RecordSnapshotSent();
|
||||
_status?.SetLastSyncUtc(_utcNow());
|
||||
// The snapshot already carries every row state up to as-of; skipping the pump past it
|
||||
// avoids re-sending tail deltas the peer would only discard via LWW.
|
||||
if (snapshotAsOfSeq > _sentThruSeq)
|
||||
// A FULL snapshot carries every row state up to as-of, so skipping the pump past it
|
||||
// avoids re-sending tail deltas the peer would only discard via LWW. A PARTIAL one
|
||||
// speaks only for its own tables, so the pump must still deliver every delta below
|
||||
// as-of — the peer has not moved its watermark either (see SnapshotApplier.OnComplete).
|
||||
if (!snapshotScope.IsPartial && snapshotAsOfSeq > _sentThruSeq)
|
||||
_sentThruSeq = snapshotAsOfSeq;
|
||||
|
||||
// A peer that cannot acknowledge the snapshot leaves only the 0.2.x option: clear once
|
||||
// the frames are on the data lane. The window that opens (enqueued, not yet delivered —
|
||||
// a crash there loses the snapshot with the debt already discharged) is why 0.3.0 peers
|
||||
// wait for the ack instead. The reconnect handshake's gap check re-detects it whenever
|
||||
// the oplog is non-empty; an EMPTY oplog remains the undetectable corner.
|
||||
if (!_peerSupportsPartialSnapshot)
|
||||
await ClearSentSnapshotDebtAsync(ct);
|
||||
}
|
||||
|
||||
await PumpLoopAsync(dataLane, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Discharges the snapshot debt this session's outbound snapshot covered, exactly once.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Taking the scope out of the field before clearing makes this idempotent, which matters
|
||||
/// because two paths reach it — the pre-0.3.0 clear-on-enqueue above and the peer's SnapshotAck
|
||||
/// — and a peer that acknowledges anyway (a newer build than its handshake claimed) must not
|
||||
/// clear a second time on top of flags a later baseline may already have re-raised.
|
||||
/// </remarks>
|
||||
private async Task ClearSentSnapshotDebtAsync(CancellationToken ct)
|
||||
{
|
||||
var scope = Interlocked.Exchange(ref _sentSnapshotScope, null);
|
||||
if (scope is null)
|
||||
return;
|
||||
await _store.ClearSnapshotDebtAsync(scope.Tables, ct);
|
||||
}
|
||||
|
||||
// The ONLY caller of duplex.Send. Control (handshake + acks) always preempts data so the peer's
|
||||
@@ -246,20 +340,54 @@ internal sealed class SyncSession
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads oplog deltas above the sent watermark and ships them, splitting each read into as many
|
||||
/// messages as <see cref="ReplicationOptions.MaxBatchBytes"/> requires.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Row count alone cannot bound a message: rows are arbitrarily wide, so any row-count cap is
|
||||
/// really a guess about the widest row, and guessing wrong produces a message the transport
|
||||
/// rejects — replication then stops converging with a size error, not a data error. Budgeting by
|
||||
/// summed serialized bytes bounds the thing that is actually limited, and lets the row cap go
|
||||
/// back to meaning what it says.
|
||||
/// </remarks>
|
||||
private async Task PumpLoopAsync(ChannelWriter<SyncMessage> dataLane, CancellationToken ct)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
// BEFORE the read, because the read itself cannot tell a pruned gap from an idle oplog.
|
||||
ThrowIfPruneGap(await _store.GetMinSeqAsync(ct));
|
||||
var batch = await _store.ReadBatchAboveAsync(_sentThruSeq, _options.MaxBatchSize, ct);
|
||||
if (batch.Count > 0)
|
||||
{
|
||||
var delta = new DeltaBatch();
|
||||
foreach (var entry in batch)
|
||||
delta.Entries.Add(ToProto(entry));
|
||||
// Blocking here on a full data lane IS the pump's backpressure.
|
||||
await dataLane.WriteAsync(new SyncMessage { DeltaBatch = delta }, ct);
|
||||
_sentThruSeq = batch[^1].Seq;
|
||||
// The exact guard on the rows about to go out, for a prune that landed between the
|
||||
// probe and the read. Free (the seq is already in hand) and strictly tighter.
|
||||
ThrowIfPruneGap(batch[0].Seq);
|
||||
var index = 0;
|
||||
while (index < batch.Count)
|
||||
{
|
||||
var delta = new DeltaBatch();
|
||||
var bytes = 0L;
|
||||
while (index < batch.Count)
|
||||
{
|
||||
var proto = ToProto(batch[index]);
|
||||
var size = EntrySize(proto);
|
||||
// The first entry of a message is taken however large it is: refusing an
|
||||
// oversized row would stall the stream on it permanently, whereas sending it
|
||||
// alone either succeeds or fails loudly at the transport with its real size.
|
||||
if (delta.Entries.Count > 0 && bytes + size > _options.MaxBatchBytes)
|
||||
break;
|
||||
delta.Entries.Add(proto);
|
||||
bytes += size;
|
||||
index++;
|
||||
}
|
||||
// Blocking here on a full data lane IS the pump's backpressure.
|
||||
await dataLane.WriteAsync(new SyncMessage { DeltaBatch = delta }, ct);
|
||||
// Advanced per MESSAGE, not per read: a fault mid-split must not re-send what
|
||||
// already went out, and must not skip what did not.
|
||||
_sentThruSeq = batch[index - 1].Seq;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -267,23 +395,77 @@ internal sealed class SyncSession
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Faults the session when the oplog horizon has climbed above this session's read position —
|
||||
/// i.e. a cap prune deleted deltas between <see cref="_sentThruSeq"/> and
|
||||
/// <paramref name="oldestAvailableSeq"/>. An empty oplog (0) is not a gap.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why a live session has to die for this.</b> <c>EnforceCapsAsync</c> sets
|
||||
/// <c>needs_snapshot</c> in the same transaction as the prune, so a RECONNECT heals — the
|
||||
/// handshake serves a full snapshot instead of deltas. Nothing, however, made the live session
|
||||
/// notice: the pump reads <c>seq > _sentThruSeq</c>, which silently resumes above the hole, and
|
||||
/// the peer acks those later seqs as if the pair were converged. On a stable LAN with a slow but
|
||||
/// alive peer that session can run for days, and the pair is durably divergent for all of it —
|
||||
/// the flag that would fix it is only ever read at handshake time. Tearing down is what turns a
|
||||
/// persisted "needs snapshot" into an actual snapshot.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why the horizon, not the flag.</b> <c>needs_snapshot</c> is also set for reasons that do
|
||||
/// not invalidate THIS session's position (and a full snapshot this session just streamed clears
|
||||
/// it only when the peer's <c>SnapshotAck</c> lands, which is after the pump has started).
|
||||
/// Comparing the horizon against the sent watermark asks the precise question instead, so a
|
||||
/// session is faulted when — and only when — rows it still owes the peer are gone.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private void ThrowIfPruneGap(long oldestAvailableSeq)
|
||||
{
|
||||
if (oldestAvailableSeq == 0 || oldestAvailableSeq <= _sentThruSeq + 1)
|
||||
return;
|
||||
throw new LocalDbOplogGapException(
|
||||
$"Oplog deltas above the sent watermark {_sentThruSeq} were pruned (oldest surviving seq " +
|
||||
$"{oldestAvailableSeq}): this session can no longer stream incrementally without skipping " +
|
||||
"them. Ending the session so the reconnect handshake serves a snapshot resync.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Serialized bytes one entry contributes to its batch message: the entry's own encoding plus
|
||||
/// the repeated field's tag and length prefix, as protobuf itself computes them.
|
||||
/// </summary>
|
||||
private static long EntrySize(OplogEntry entry) =>
|
||||
1 + Google.Protobuf.CodedOutputStream.ComputeMessageSize(entry);
|
||||
|
||||
private async Task ReceiveLoopAsync(
|
||||
SyncDuplex duplex, ChannelWriter<SyncMessage> controlLane, CancellationToken ct)
|
||||
{
|
||||
while (await duplex.Inbox.WaitToReadAsync(ct))
|
||||
// One-message lookahead. Coalescing means reading ahead until a message turns out NOT to be
|
||||
// of the kind being coalesced, and a Channel cannot un-read — so that message is parked here
|
||||
// and processed first on the next pass. Its presence also keeps the outer loop from waiting
|
||||
// on the inbox when there is already work in hand.
|
||||
SyncMessage? lookahead = null;
|
||||
|
||||
while (lookahead is not null || await duplex.Inbox.WaitToReadAsync(ct))
|
||||
{
|
||||
while (duplex.Inbox.TryRead(out var msg))
|
||||
while (true)
|
||||
{
|
||||
var msg = lookahead;
|
||||
lookahead = null;
|
||||
if (msg is null)
|
||||
{
|
||||
if (!duplex.Inbox.TryRead(out var read))
|
||||
break;
|
||||
msg = read;
|
||||
}
|
||||
|
||||
switch (msg.MsgCase)
|
||||
{
|
||||
case SyncMessage.MsgOneofCase.DeltaBatch:
|
||||
// Deltas apply normally even mid-snapshot: LWW makes the interleave safe.
|
||||
await HandleDeltaBatchAsync(controlLane, msg.DeltaBatch, ct);
|
||||
lookahead = await HandleCoalescedDeltasAsync(duplex, controlLane, msg.DeltaBatch, ct);
|
||||
break;
|
||||
case SyncMessage.MsgOneofCase.DeltaAck:
|
||||
var acked = msg.DeltaAck.AppliedThruSeq;
|
||||
InterlockedMax(ref _peerAckedSeq, acked);
|
||||
await _store.RecordPeerAckAsync(acked, ct);
|
||||
lookahead = await HandleCoalescedAcksAsync(duplex, msg.DeltaAck, ct);
|
||||
break;
|
||||
case SyncMessage.MsgOneofCase.SnapshotBegin:
|
||||
if (SnapshotApplier is null)
|
||||
@@ -305,6 +487,20 @@ internal sealed class SyncSession
|
||||
_snapshotReceiving = false;
|
||||
_metrics?.RecordSnapshotReceived();
|
||||
_status?.SetLastSyncUtc(_utcNow());
|
||||
// Acknowledge only to a peer that asked for it. Sent AFTER the applier's
|
||||
// durable watermark write, so the ack means "I hold these rows", which is
|
||||
// what lets the sender drop its per-table flags. On the unbounded control
|
||||
// lane, like DeltaAck: an ack must never queue behind a data backlog.
|
||||
if (_peerSupportsPartialSnapshot)
|
||||
await controlLane.WriteAsync(
|
||||
new SyncMessage { SnapshotAck = new SnapshotAck { AsOfSeq = msg.SnapshotComplete.AsOfSeq } }, ct);
|
||||
break;
|
||||
case SyncMessage.MsgOneofCase.SnapshotAck:
|
||||
// The peer has durably applied the snapshot THIS node streamed, so the
|
||||
// tables it covered no longer owe one. Nothing to reconcile against the
|
||||
// ack's as-of seq: only one snapshot is ever sent per session, so the scope
|
||||
// held in _sentSnapshotScope is unambiguously the one being acknowledged.
|
||||
await ClearSentSnapshotDebtAsync(ct);
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException(
|
||||
@@ -314,19 +510,98 @@ internal sealed class SyncSession
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleDeltaBatchAsync(ChannelWriter<SyncMessage> controlLane, DeltaBatch batch, CancellationToken ct)
|
||||
/// <summary>
|
||||
/// Applies the given delta batch together with any delta batches ALREADY waiting in the inbox,
|
||||
/// as one transaction answered by one ack; returns the non-delta message that ended the run (to
|
||||
/// be processed next), or null if the inbox merely ran dry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Byte-budgeting the sender produces more, smaller messages, and one apply transaction plus one
|
||||
/// ack round-trip per message is the cost that would otherwise scale with that. Coalescing pays
|
||||
/// it once per burst instead — and a burst is exactly what a reconnect after an outage delivers.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Only messages already queued are absorbed (<c>TryRead</c>, never a wait), so this adds no
|
||||
/// latency: a trickle of one batch at a time coalesces nothing and behaves exactly as before.
|
||||
/// The run stops at the sender's own caps, so the extra memory held is bounded by roughly one
|
||||
/// sender batch beyond the budget — the overshoot of finishing whichever batch crosses it.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task<SyncMessage?> HandleCoalescedDeltasAsync(
|
||||
SyncDuplex duplex, ChannelWriter<SyncMessage> controlLane, DeltaBatch first, CancellationToken ct)
|
||||
{
|
||||
if (batch.Entries.Count == 0)
|
||||
var entries = new List<OplogEntry>(first.Entries);
|
||||
var bytes = 0L;
|
||||
foreach (var entry in first.Entries)
|
||||
bytes += EntrySize(entry);
|
||||
|
||||
SyncMessage? lookahead = null;
|
||||
while (entries.Count < _options.MaxBatchSize && bytes < _options.MaxBatchBytes
|
||||
&& duplex.Inbox.TryRead(out var next))
|
||||
{
|
||||
if (next.MsgCase != SyncMessage.MsgOneofCase.DeltaBatch)
|
||||
{
|
||||
lookahead = next;
|
||||
break;
|
||||
}
|
||||
foreach (var entry in next.DeltaBatch.Entries)
|
||||
{
|
||||
entries.Add(entry);
|
||||
bytes += EntrySize(entry);
|
||||
}
|
||||
}
|
||||
|
||||
await ApplyDeltaEntriesAsync(controlLane, entries, ct);
|
||||
return lookahead;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records the given ack together with any acks ALREADY waiting in the inbox, as a single
|
||||
/// watermark write; returns the non-ack message that ended the run, or null if the inbox ran dry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Exactly equivalent to recording each in turn, and strictly cheaper: <c>applied_thru_seq</c> is
|
||||
/// monotonic and <c>RecordPeerAckAsync</c> is MAX-guarded, so every superseded ack would have
|
||||
/// written a watermark that the final one overwrites and run a DELETE below its own horizon.
|
||||
/// Unbounded coalescing is safe here — acks collapse into one number, accumulating nothing.
|
||||
/// </remarks>
|
||||
private async Task<SyncMessage?> HandleCoalescedAcksAsync(
|
||||
SyncDuplex duplex, DeltaAck first, CancellationToken ct)
|
||||
{
|
||||
var acked = first.AppliedThruSeq;
|
||||
|
||||
SyncMessage? lookahead = null;
|
||||
while (duplex.Inbox.TryRead(out var next))
|
||||
{
|
||||
if (next.MsgCase != SyncMessage.MsgOneofCase.DeltaAck)
|
||||
{
|
||||
lookahead = next;
|
||||
break;
|
||||
}
|
||||
if (next.DeltaAck.AppliedThruSeq > acked)
|
||||
acked = next.DeltaAck.AppliedThruSeq;
|
||||
}
|
||||
|
||||
InterlockedMax(ref _peerAckedSeq, acked);
|
||||
await _store.RecordPeerAckAsync(acked, ct);
|
||||
return lookahead;
|
||||
}
|
||||
|
||||
private async Task ApplyDeltaEntriesAsync(
|
||||
ChannelWriter<SyncMessage> controlLane, IReadOnlyList<OplogEntry> batchEntries, CancellationToken ct)
|
||||
{
|
||||
if (batchEntries.Count == 0)
|
||||
return;
|
||||
|
||||
var now = _utcNow();
|
||||
var nowMs = now.ToUnixTimeMilliseconds();
|
||||
var driftLimitMs = nowMs + (long)_options.MaxHlcDriftAhead.TotalMilliseconds;
|
||||
var toApply = new List<OplogEntryRecord>(batch.Entries.Count);
|
||||
var toApply = new List<OplogEntryRecord>(batchEntries.Count);
|
||||
var maxSeq = 0L;
|
||||
var driftDeadLettered = 0;
|
||||
|
||||
foreach (var proto in batch.Entries)
|
||||
foreach (var proto in batchEntries)
|
||||
{
|
||||
if (proto.Seq > maxSeq) maxSeq = proto.Seq;
|
||||
var entry = FromProto(proto);
|
||||
@@ -407,6 +682,9 @@ internal sealed class SyncSession
|
||||
NodeId = _db.NodeId,
|
||||
LibSchemaVersion = LibSchemaVersion,
|
||||
LastAppliedRemoteSeq = peerState.LastAppliedRemoteSeq,
|
||||
// Unconditional: this build always understands a scoped snapshot and always answers a
|
||||
// completed one. It is a statement about the code, not about configuration or state.
|
||||
SupportsPartialSnapshot = true,
|
||||
};
|
||||
foreach (var table in _db.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal))
|
||||
handshake.Tables.Add(new TableDigest { TableName = table.Name, Digest = table.Digest });
|
||||
@@ -437,11 +715,63 @@ internal sealed class SyncSession
|
||||
"Replication handshake rejected (fail-closed): " + string.Join("; ", parts) + ".");
|
||||
}
|
||||
|
||||
private async Task<bool> ComputeSnapshotRequiredAsync(Handshake peerHandshake, PeerState peerState, CancellationToken ct)
|
||||
/// <summary>
|
||||
/// Decides what snapshot this node owes the peer: null for none, <see cref="SnapshotScope.Full"/>
|
||||
/// for every registered table, or a scoped snapshot naming just the tables that need one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The two sources of snapshot debt are genuinely different in reach, which is why they are
|
||||
/// stored separately. An <b>oplog-horizon</b> gap — the peer's watermark below the oldest delta
|
||||
/// still streamable, which is what a cap prune or an impossible peer ack leaves behind — means
|
||||
/// deltas have been lost for the oplog as a whole, so nothing short of every table is sound. A
|
||||
/// <b>baseline</b> means one table's pre-existing rows were never captured, and says nothing at
|
||||
/// all about the others.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A scope that happens to name every registered table is returned as Full rather than as a
|
||||
/// list: it covers exactly what a full snapshot covers, so it should also get a full snapshot's
|
||||
/// watermark advance on both sides, and it needs no promotion for a pre-0.3.0 peer.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Flags naming tables that are not registered are ignored (they can only stream nothing).
|
||||
/// <c>DeregisterReplicated</c> drops them with the table's ledger rows, so the surviving way to
|
||||
/// hold one is to drop the SQL table without deregistering — which the handshake's fail-closed
|
||||
/// digest comparison rejects for its own reasons.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task<SnapshotScope?> ComputeSnapshotScopeAsync(Handshake peerHandshake, PeerState peerState, CancellationToken ct)
|
||||
{
|
||||
if (peerState.NeedsSnapshot)
|
||||
return true;
|
||||
if (peerState.NeedsSnapshot || await HasHorizonGapAsync(peerHandshake, peerState, ct))
|
||||
return SnapshotScope.Full;
|
||||
|
||||
var flagged = await _store.GetTablesNeedingSnapshotAsync(ct);
|
||||
if (flagged.Count == 0)
|
||||
return null;
|
||||
|
||||
var scoped = flagged.Where(_db.ReplicatedTables.ContainsKey).ToList();
|
||||
if (scoped.Count == 0)
|
||||
return null;
|
||||
if (scoped.Count >= _db.ReplicatedTables.Count)
|
||||
return SnapshotScope.Full;
|
||||
|
||||
// A peer that would read SnapshotBegin.tables as "all tables" and then advance its watermark
|
||||
// past deltas this snapshot did not carry gets the full snapshot instead. Wider than asked
|
||||
// for, but it is the only shape the older protocol can express correctly — and it is what
|
||||
// 0.2.x did for every baseline.
|
||||
if (!_peerSupportsPartialSnapshot)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Peer {PeerNodeId} does not support scoped snapshots; promoting the snapshot of {Tables} to a full one.",
|
||||
PeerNodeId, string.Join(",", scoped));
|
||||
return SnapshotScope.Full;
|
||||
}
|
||||
|
||||
return new SnapshotScope(scoped);
|
||||
}
|
||||
|
||||
private async Task<bool> HasHorizonGapAsync(Handshake peerHandshake, PeerState peerState, CancellationToken ct)
|
||||
{
|
||||
var minRows = await _db.QueryAsync(
|
||||
"SELECT COALESCE(MIN(seq), 0) FROM __localdb_oplog", static r => r.GetInt64(0), null, ct);
|
||||
var minSeq = minRows[0];
|
||||
|
||||
@@ -6,7 +6,7 @@ namespace ZB.MOM.WW.LocalDb.Replication.Internal;
|
||||
/// <summary>
|
||||
/// Shared construction of a <see cref="SyncSession"/> and its dependencies (OplogStore + LwwApplier)
|
||||
/// from a <see cref="SqliteLocalDb"/> and the bound options. Both the service and client adapters use
|
||||
/// this so they build an identical session; snapshot hooks are left null (Task 12 fills them in).
|
||||
/// this so they build an identical session.
|
||||
/// </summary>
|
||||
internal static class SyncSessionFactory
|
||||
{
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace ZB.MOM.WW.LocalDb.Replication;
|
||||
|
||||
/// <summary>
|
||||
/// Thrown by a live sync session when the oplog horizon has moved ABOVE the position that session
|
||||
/// is streaming from: a cap prune deleted deltas it had not yet sent, so every later delta it could
|
||||
/// ship would land on the far side of a hole the peer will never receive.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is not a transport failure and not a data error — it is the session discovering that its own
|
||||
/// incremental position has become unsound. Nothing can repair it in-session (the deleted rows are
|
||||
/// gone), so the session tears itself down; the same prune set <c>needs_snapshot</c>, which makes the
|
||||
/// reconnect handshake serve a snapshot instead of deltas. Catching this and continuing would
|
||||
/// reintroduce exactly the silent divergence it exists to stop.
|
||||
/// </remarks>
|
||||
public sealed class LocalDbOplogGapException : Exception
|
||||
{
|
||||
/// <summary>Creates the exception with a message describing the gap.</summary>
|
||||
public LocalDbOplogGapException(string message) : base(message) { }
|
||||
|
||||
/// <summary>Creates the exception with a message and the underlying cause.</summary>
|
||||
public LocalDbOplogGapException(string message, Exception innerException)
|
||||
: base(message, innerException) { }
|
||||
}
|
||||
@@ -21,8 +21,26 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
|
||||
private readonly LocalDbMetrics? _metrics;
|
||||
private readonly SyncStatus? _status;
|
||||
|
||||
// 0 = idle, 1 = a stream is active. Interlocked-guarded across concurrent calls (singleton service).
|
||||
private int _active;
|
||||
// The stream currently holding the guard, or null when idle. Guarded by _gate (singleton service).
|
||||
private readonly object _gate = new();
|
||||
private Registration? _active;
|
||||
|
||||
/// <summary>
|
||||
/// How long a displacing call waits for the incumbent it evicted to actually finish before
|
||||
/// giving up and refusing itself. Test seam; the default is generous because eviction only has
|
||||
/// to outrun the operator, not the network.
|
||||
/// </summary>
|
||||
internal TimeSpan TakeoverTimeout { get; set; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>One in-flight <c>Sync</c> handler: its cancellation handle plus a completion signal
|
||||
/// a displacing handler can await, so two sessions never run against the database at once.</summary>
|
||||
private sealed class Registration(CancellationTokenSource cts)
|
||||
{
|
||||
public CancellationTokenSource Cts { get; } = cts;
|
||||
|
||||
public TaskCompletionSource Finished { get; } =
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the passive sync service. <paramref name="status"/> is taken as the public
|
||||
@@ -41,21 +59,47 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
|
||||
_status = status as SyncStatus;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Accepts one inbound sync stream, evicting any stream already holding the guard.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Newest wins.</b> The guard exists to SERIALIZE sessions — a node syncs with exactly one
|
||||
/// peer, and two sessions against one database would fight over the same watermarks — not to
|
||||
/// fence out an impostor; authentication is the interceptor's job, and an authenticated caller
|
||||
/// on this key IS the peer by definition. Refusing the newcomer therefore protected nothing and
|
||||
/// cost everything in the case that actually happens: the dialling node crashes or is
|
||||
/// partitioned, this handler stays parked forever inside <c>MoveNext</c> on a half-open stream
|
||||
/// that will never produce another byte, and every reconnect from the rebooted peer is rejected
|
||||
/// <c>AlreadyExists</c> — potentially for as long as the pair stays idle, while this side keeps
|
||||
/// reporting itself connected. The newest dialler is the one with a live link, so it takes over.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Eviction is cancel-then-await, not cancel-and-go: the incumbent's cancellation unwinds its
|
||||
/// session and reader pump, and the newcomer waits for that to complete before starting its own,
|
||||
/// so the "one session at a time" property the guard was built for still holds. An incumbent
|
||||
/// that cannot be evicted within <see cref="TakeoverTimeout"/> is the one case where the old
|
||||
/// answer is still right — the newcomer refuses itself rather than double-running.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public override async Task Sync(
|
||||
IAsyncStreamReader<SyncMessage> requestStream,
|
||||
IServerStreamWriter<SyncMessage> responseStream,
|
||||
ServerCallContext context)
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _active, 1, 0) != 0)
|
||||
throw new RpcException(new Status(StatusCode.AlreadyExists, "a sync stream is already active"));
|
||||
// The linked source exists so the reader pump is deterministically torn down on EVERY exit
|
||||
// path — including a session fault for a local (non-transport) reason, where the call token
|
||||
// never fires and the pump would otherwise keep reading a stream whose handler has exited.
|
||||
// It is also the handle a later call cancels to displace this one.
|
||||
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken);
|
||||
var mine = new Registration(linkedCts);
|
||||
|
||||
try
|
||||
{
|
||||
// The linked source exists so the reader pump is deterministically torn down on EVERY
|
||||
// exit path — including a session fault for a local (non-transport) reason, where the
|
||||
// call token never fires and the pump would otherwise keep reading a stream whose
|
||||
// handler has exited.
|
||||
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken);
|
||||
// Inside the try so the finally below always releases the guard — including when the
|
||||
// takeover itself is abandoned because THIS caller gave up mid-handover.
|
||||
await TakeOverAsync(mine, context.CancellationToken);
|
||||
|
||||
var (duplex, readerTask) = GrpcSyncDuplex.Create(requestStream, responseStream, linkedCts.Token);
|
||||
var session = SyncSessionFactory.Create(_db, _options, _loggerFactory, _metrics, _status);
|
||||
_status?.SessionStarted();
|
||||
@@ -81,13 +125,66 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
|
||||
{
|
||||
// Client disconnect / server shutdown: the normal end of a long-lived streaming RPC.
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Displaced by a newer stream: an orderly handover, not a failure of this node.
|
||||
_logger.LogInformation("Sync stream ended: displaced by a newer stream from the peer.");
|
||||
}
|
||||
catch (LocalDbSchemaMismatchException ex)
|
||||
{
|
||||
throw new RpcException(new Status(StatusCode.FailedPrecondition, ex.Message));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Exchange(ref _active, 0);
|
||||
lock (_gate)
|
||||
{
|
||||
if (ReferenceEquals(_active, mine))
|
||||
_active = null;
|
||||
}
|
||||
mine.Finished.TrySetResult();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Claims the single-stream guard for <paramref name="mine"/>, cancelling and awaiting whatever
|
||||
/// held it. Throws <c>AlreadyExists</c> only when the incumbent refuses to end.
|
||||
/// </summary>
|
||||
private async Task TakeOverAsync(Registration mine, CancellationToken ct)
|
||||
{
|
||||
Registration? displaced;
|
||||
lock (_gate)
|
||||
{
|
||||
displaced = _active;
|
||||
_active = mine;
|
||||
}
|
||||
if (displaced is null)
|
||||
return;
|
||||
|
||||
_logger.LogWarning(
|
||||
"A new sync stream arrived while one was already active; displacing the incumbent (newest wins).");
|
||||
try
|
||||
{
|
||||
// The incumbent may already be disposing its own source between its finally and here.
|
||||
displaced.Cts.Cancel();
|
||||
}
|
||||
catch (ObjectDisposedException) { }
|
||||
|
||||
try
|
||||
{
|
||||
await displaced.Finished.Task.WaitAsync(TakeoverTimeout, ct);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
// Could not evict it: two live sessions over one database is worse than a refused dial,
|
||||
// so fall back to the historical answer — and hand the guard back to the incumbent.
|
||||
lock (_gate)
|
||||
{
|
||||
if (ReferenceEquals(_active, mine))
|
||||
_active = displaced;
|
||||
}
|
||||
throw new RpcException(new Status(
|
||||
StatusCode.AlreadyExists,
|
||||
"a sync stream is already active and did not yield within the takeover timeout"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ public sealed class MaintenanceBackgroundService : BackgroundService
|
||||
{
|
||||
_snapshotFlagWarned = true;
|
||||
_logger.LogWarning(
|
||||
"Oplog backlog/age caps exceeded: pruned to the ceiling and flagged needs_snapshot — the peer must snapshot-resync. Not re-logged until a clean tick.");
|
||||
"Oplog backlog/age caps exceeded: pruned to the ceiling and flagged needs_snapshot — the peer must snapshot-resync. A session streaming from below the new horizon ends itself on its next pump tick so the reconnect serves that snapshot. Not re-logged until a clean tick.");
|
||||
}
|
||||
else if (!snapshotFlagged)
|
||||
{
|
||||
|
||||
@@ -14,9 +14,44 @@ public sealed class ReplicationOptions
|
||||
/// <summary>How often the change pump flushes pending oplog deltas to the peer.</summary>
|
||||
public TimeSpan FlushInterval { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>Maximum oplog entries sent per outbound batch.</summary>
|
||||
/// <summary>
|
||||
/// Secondary, row-count cap on an outbound batch (and the snapshot streamer's DB page size).
|
||||
/// <see cref="MaxBatchBytes"/> is the primary cap.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Historically this was the ONLY cap, which made it a proxy for a byte budget that consumers
|
||||
/// had to tune by hand against their widest row — ScadaBridge pinned it to 16 because a 70 KB
|
||||
/// <c>config_json</c> times the 500 default is ~35 MB against a 4 MB gRPC limit. With
|
||||
/// <see cref="MaxBatchBytes"/> doing that job directly, this is back to being what its name
|
||||
/// says: a bound on how many rows one message may carry.
|
||||
/// </remarks>
|
||||
public int MaxBatchSize { get; set; } = 500;
|
||||
|
||||
/// <summary>
|
||||
/// Primary cap on an outbound batch: the summed serialized size of the entries in one
|
||||
/// <c>DeltaBatch</c> / <c>SnapshotBatch</c> message, in bytes. Default 2 MB.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Sized at half the default 4 MB gRPC receive limit, so the message framing, the oneof wrapper
|
||||
/// and any per-entry overhead the estimate does not model all fit inside the margin.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>A single row larger than the budget is still sent, alone.</b> Skipping it would stall the
|
||||
/// stream permanently on one oversized row; sending it gives the transport the chance to accept
|
||||
/// it (gRPC limits are configurable) and, if it refuses, a loud transport error naming the size
|
||||
/// rather than silent non-convergence.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Wire compatibility:</b> none required. Batch size is a sender-side choice the protocol
|
||||
/// already leaves free — <c>DeltaBatch</c>/<c>SnapshotBatch</c> carry a repeated field with no
|
||||
/// declared count — so a node on this version interoperates with a peer on any earlier one in
|
||||
/// both directions, with no proto change and no <c>lib_schema_version</c> bump. (ScadaBridge
|
||||
/// site pairs stop and start together regardless; this simply does not depend on it.)
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public int MaxBatchBytes { get; set; } = 2 * 1024 * 1024;
|
||||
|
||||
/// <summary>Backlog ceiling: exceeding it flags a snapshot resync and prunes the oldest oplog rows.</summary>
|
||||
public long MaxOplogRows { get; set; } = 1_000_000;
|
||||
|
||||
@@ -29,6 +64,34 @@ public sealed class ReplicationOptions
|
||||
/// <summary>Upper bound on the exponential reconnect backoff.</summary>
|
||||
public TimeSpan ReconnectBackoffMax { get; set; } = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>
|
||||
/// How long the initiator's HTTP/2 connection may sit without a keepalive PING before one is
|
||||
/// sent. Default 60s.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// A sync stream is long-lived and, on a converged pair, almost entirely silent — which is
|
||||
/// exactly the shape a half-open TCP connection hides behind. Without pings, a hard crash or a
|
||||
/// partition of one node leaves the other holding a stream that will never produce another byte
|
||||
/// and never fail either: the initiator sits in a session it believes is healthy, and the
|
||||
/// passive node's handler sits inside its single-stream guard, refusing the reconnect that would
|
||||
/// heal the pair. Pings turn that into a detected failure on both ends.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Sent with <c>KeepAlivePingPolicy.Always</c>, so they flow on an idle connection too — an
|
||||
/// active-streams-only policy would not fire in precisely the silent-pair case above. Safe over
|
||||
/// h2c: a PING is an HTTP/2 frame, nothing to do with TLS.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public TimeSpan KeepAlivePingDelay { get; set; } = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>
|
||||
/// How long the initiator waits for a keepalive PING ack before tearing the connection down.
|
||||
/// Default 20s. Detection therefore lands within roughly
|
||||
/// <see cref="KeepAlivePingDelay"/> + this.
|
||||
/// </summary>
|
||||
public TimeSpan KeepAlivePingTimeout { get; set; } = TimeSpan.FromSeconds(20);
|
||||
|
||||
/// <summary>How often the maintenance service enforces the oplog caps and prunes expired tombstones.</summary>
|
||||
public TimeSpan MaintenanceInterval { get; set; } = TimeSpan.FromSeconds(60);
|
||||
|
||||
|
||||
@@ -28,6 +28,8 @@ public sealed class ReplicationOptionsValidator : IValidateOptions<ReplicationOp
|
||||
failures.Add($"LocalDb:Replication:FlushInterval must be greater than zero; got {options.FlushInterval}.");
|
||||
if (options.MaxBatchSize <= 0)
|
||||
failures.Add($"LocalDb:Replication:MaxBatchSize must be greater than 0; got {options.MaxBatchSize}.");
|
||||
if (options.MaxBatchBytes <= 0)
|
||||
failures.Add($"LocalDb:Replication:MaxBatchBytes must be greater than 0; got {options.MaxBatchBytes}.");
|
||||
if (options.MaxOplogRows <= 0)
|
||||
failures.Add($"LocalDb:Replication:MaxOplogRows must be greater than 0; got {options.MaxOplogRows}.");
|
||||
if (options.MaxOplogAge <= TimeSpan.Zero)
|
||||
@@ -38,6 +40,10 @@ public sealed class ReplicationOptionsValidator : IValidateOptions<ReplicationOp
|
||||
failures.Add($"LocalDb:Replication:ReconnectBackoffMax must be greater than zero; got {options.ReconnectBackoffMax}.");
|
||||
else if (options.ReconnectBackoffMax > ReconnectBackoffCeiling)
|
||||
failures.Add($"LocalDb:Replication:ReconnectBackoffMax must be at most {ReconnectBackoffCeiling}; got {options.ReconnectBackoffMax}.");
|
||||
if (options.KeepAlivePingDelay <= TimeSpan.Zero)
|
||||
failures.Add($"LocalDb:Replication:KeepAlivePingDelay must be greater than zero; got {options.KeepAlivePingDelay}.");
|
||||
if (options.KeepAlivePingTimeout <= TimeSpan.Zero)
|
||||
failures.Add($"LocalDb:Replication:KeepAlivePingTimeout must be greater than zero; got {options.KeepAlivePingTimeout}.");
|
||||
if (options.MaintenanceInterval <= TimeSpan.Zero)
|
||||
failures.Add($"LocalDb:Replication:MaintenanceInterval must be greater than zero; got {options.MaintenanceInterval}.");
|
||||
if (options.MaxHlcDriftAhead <= TimeSpan.Zero)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Net.Http;
|
||||
using Grpc.Core;
|
||||
using Grpc.Net.Client;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
@@ -76,7 +77,7 @@ public sealed class SyncBackgroundService : BackgroundService
|
||||
{
|
||||
var channel = ChannelFactory is { } factory
|
||||
? factory()
|
||||
: ownedChannel = GrpcChannel.ForAddress(_options.PeerAddress);
|
||||
: ownedChannel = DialPeer();
|
||||
|
||||
var client = new LocalDbSync.LocalDbSyncClient(channel);
|
||||
var headers = new Metadata();
|
||||
@@ -152,6 +153,39 @@ public sealed class SyncBackgroundService : BackgroundService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Dials the peer over a channel that sends HTTP/2 keepalive PINGs, so a half-open link is
|
||||
/// detected instead of being waited on forever.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The bare <c>GrpcChannel.ForAddress(address)</c> this replaces had NO keepalive of any kind,
|
||||
/// and neither end of a sync stream ever times out on silence by design (a converged pair is
|
||||
/// legitimately quiet). A hard crash or partition of the peer therefore left this node inside a
|
||||
/// session that could never make progress and could never fail — and its own half of that stream
|
||||
/// pinned on the peer, where the single-stream guard turns it into a refusal of the very
|
||||
/// reconnect that heals the pair.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <c>DisposeHttpClient</c> hands the handler's lifetime to the channel, which this loop already
|
||||
/// disposes on every exit path.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private GrpcChannel DialPeer() =>
|
||||
GrpcChannel.ForAddress(_options.PeerAddress, new GrpcChannelOptions
|
||||
{
|
||||
HttpHandler = new SocketsHttpHandler
|
||||
{
|
||||
KeepAlivePingDelay = _options.KeepAlivePingDelay,
|
||||
KeepAlivePingTimeout = _options.KeepAlivePingTimeout,
|
||||
// Always, not WithActiveRequests: the case being closed is a SILENT stream on a dead
|
||||
// link, and an active-requests policy stops pinging once the peer stops responding.
|
||||
KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always,
|
||||
EnableMultipleHttp2Connections = true,
|
||||
},
|
||||
DisposeHttpClient = true,
|
||||
});
|
||||
|
||||
private static bool IsSchemaMismatch(Exception? ex) =>
|
||||
ex is LocalDbSchemaMismatchException
|
||||
|| (ex is RpcException rpc && rpc.StatusCode == StatusCode.FailedPrecondition);
|
||||
|
||||
@@ -38,10 +38,133 @@ public interface ILocalDb
|
||||
/// schema digest, and installs the three AFTER triggers that capture every change into the
|
||||
/// oplog + row-version tables within the same transaction as the consumer's write. Idempotent
|
||||
/// (re-registering regenerates the triggers). Throws <see cref="LocalDbRegistrationException"/>
|
||||
/// if the name is unsafe, the table is missing, or it has no primary key.
|
||||
/// if the name is unsafe, the table is missing, it has no primary key, it has a BLOB column, or
|
||||
/// another table references it with a row-mutating <c>ON DELETE</c> action.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Replicated tables must not be cascade-delete foreign-key parents.</b> Applying a remote row
|
||||
/// uses <c>INSERT OR REPLACE</c>, whose implicit DELETE fires <c>ON DELETE CASCADE</c> /
|
||||
/// <c>SET NULL</c> / <c>SET DEFAULT</c> on referencing tables while change capture is suppressed
|
||||
/// — the child rows it touches are never captured, so the peer keeps them and the pair diverges
|
||||
/// silently. Such a table is refused here, because registration is the only point at which the
|
||||
/// problem is still visible. <c>NO ACTION</c> / <c>RESTRICT</c> are allowed: they mutate no child
|
||||
/// row, so the implicit delete simply fails and the applier dead-letters the entry.
|
||||
/// </remarks>
|
||||
ReplicatedTable RegisterReplicated(string tableName);
|
||||
|
||||
/// <summary>
|
||||
/// As <see cref="RegisterReplicated(string)"/>, additionally baselining rows that already exist
|
||||
/// in the table when <paramref name="baselineExistingRows"/> is true.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The table to opt into replication.</param>
|
||||
/// <param name="baselineExistingRows">
|
||||
/// When true, every row already in the table that has no <c>__localdb_row_version</c> entry is
|
||||
/// given one at the <b>baseline floor</b> — HLC <c>0</c>, stamped with this node's id — and, if
|
||||
/// anything was seeded, a snapshot resync is flagged <b>for this table</b> so the peer actually
|
||||
/// receives them.
|
||||
/// </param>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>The problem this solves.</b> Capture is change-data-capture: without baselining, a row
|
||||
/// written before registration exists in neither the oplog nor the row-version ledger, and the
|
||||
/// snapshot streamer pages from the LEDGER — so a node that starts replicating LATE never sends
|
||||
/// its pre-existing rows, silently and forever. Opting in seeds the ledger so a snapshot can
|
||||
/// carry them.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why the floor is HLC 0, and why that is LWW-safe.</b> A seeded version must never beat a
|
||||
/// genuine write. Every genuine HLC is a UTC millisecond shifted left 16 bits, so every genuine
|
||||
/// HLC is greater than 0: a baselined row therefore LOSES every comparison against a real remote
|
||||
/// write of the same key, and wins only where the peer has no version of that key at all
|
||||
/// (which is precisely the case baselining exists to fix). It cannot perturb the local clock
|
||||
/// either — clock recovery takes a MAX over existing HLCs, and 0 raises nothing.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The one genuinely ambiguous case</b> is both nodes baselining the SAME key (e.g. both
|
||||
/// restored the same legacy file). Both sides hold HLC 0, so the LWW tie-break falls to the
|
||||
/// node-id ordinal: deterministic and convergent, but arbitrary as to content. Nothing can do
|
||||
/// better — two rows of unknown, unordered antiquity carry no information about which is newer.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Idempotent.</b> Seeding uses <c>ON CONFLICT DO NOTHING</c>, so a row that already has a
|
||||
/// ledger entry — captured normally, or seeded on an earlier boot — keeps its genuine version
|
||||
/// and no snapshot is flagged. Passing true on every boot is therefore free after the first.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Cost, and what bounds it (0.3.0).</b> The snapshot flag is <b>per-table</b>
|
||||
/// (<c>__localdb_snapshot_state</c>, on-disk schema v3), so baselining one table asks the peer
|
||||
/// for that table alone — the tables around it are not re-streamed. Through 0.2.x the flag was a
|
||||
/// single per-database bit and baselining anything re-sent everything, in both directions at
|
||||
/// once on a pair enabling replication late.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Two cases still stream every registered table, by design. A <b>pre-0.3.0 peer</b> cannot be
|
||||
/// told a snapshot is partial (it would advance its watermark past deltas it never received),
|
||||
/// so a scoped snapshot is promoted to a full one for it. And the <b>oplog-horizon</b> paths —
|
||||
/// a cap prune, an impossible peer ack — invalidate delta continuity for every table at once
|
||||
/// and still set the database-wide flag. So the operational advice survives, narrowed: baseline
|
||||
/// a high-volume table (an event log at its retention cap, say) deliberately, and <b>purge or
|
||||
/// trim it first</b>, because that ONE table's rows still travel in full. The work is bounded
|
||||
/// either way — outbound messages respect <c>MaxBatchBytes</c>, the receive side is bounded by
|
||||
/// the transport's inbox.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Crash safety.</b> Between 0.3.0 peers a table's flag is cleared only when the receiver
|
||||
/// acknowledges the completed snapshot, so an ungraceful exit part-way through re-streams the
|
||||
/// same scope on reconnect rather than losing it. Against a pre-0.3.0 peer (which sends no such
|
||||
/// acknowledgement) the 0.2.x behaviour is kept: the flag clears once the snapshot is on the
|
||||
/// wire, and the handshake's horizon gap check is the net.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
ReplicatedTable RegisterReplicated(string tableName, bool baselineExistingRows);
|
||||
|
||||
/// <summary>
|
||||
/// Removes a table from replication: drops its three capture triggers and prunes its rows from
|
||||
/// <c>__localdb_oplog</c> and <c>__localdb_row_version</c>, in one transaction. Idempotent, and
|
||||
/// deliberately usable on a table this process never registered — cleaning up triggers left
|
||||
/// behind by an EARLIER build is the primary use.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The table to stop capturing. The table's own data is never touched.</param>
|
||||
/// <returns>True if anything was removed (a trigger dropped or a row pruned); false if there was nothing to clean up.</returns>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Intended caller:</b> an application that has decided this node no longer replicates (or no
|
||||
/// longer replicates this table). Capture is not free — the three triggers run inside every
|
||||
/// consumer write transaction and append to an oplog nothing will ever drain — so a node with no
|
||||
/// peer should not be paying for it.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Pruning the row-version ledger is safe with respect to re-registration</b>, because
|
||||
/// <see cref="RegisterReplicated(string, bool)"/> can re-baseline. Without baselining, a
|
||||
/// deregister-then-register cycle would leave the previously-known rows unstreamable — the same
|
||||
/// late-opt-in hole — so the two APIs are designed as a pair.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Deregistration must be symmetric across a replicating pair, and the library enforces it
|
||||
/// loudly.</b> The sync handshake compares the two nodes' registered-table digests fail-closed,
|
||||
/// so a node that drops a table its peer still replicates simply stops syncing — with a
|
||||
/// schema-mismatch error, not silent divergence. Deregister on both nodes, or on neither.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The HLC's durable crash anchor is flushed as part of the same transaction</b>, before the
|
||||
/// prune. The clock recovers on open as the MAX of <c>hlc_high_water</c>, the oplog's HLCs, the
|
||||
/// row-version ledger's HLCs and the last-seen peer HLC — and <c>hlc_high_water</c> is otherwise
|
||||
/// written only on graceful disposal. Since this API deletes the deregistered table's oplog and
|
||||
/// ledger rows, a file with no other registered table would, after an UNGRACEFUL exit (crash,
|
||||
/// SIGKILL, power loss), have nothing left to recover from: the clock would restart from
|
||||
/// wall-clock time and could re-issue stamps below HLCs peers already hold, which LWW then
|
||||
/// discards as older. Flushing here removes that window.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Backlog-depth residual.</b> Pruning one table's rows can leave holes in the oplog's seq
|
||||
/// range, and the O(1) depth estimate assumes the retained range is contiguous, so it can
|
||||
/// over-report until the peer acks past the holes. Over-reporting is conservative — it can only
|
||||
/// trigger an earlier snapshot resync, never suppress one — and on the node this API is for
|
||||
/// (no peer at all) depth is moot.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
bool DeregisterReplicated(string tableName);
|
||||
|
||||
/// <summary>The tables currently opted into replication, keyed by table name.</summary>
|
||||
IReadOnlyDictionary<string, ReplicatedTable> ReplicatedTables { get; }
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.LocalDb.Internal;
|
||||
@@ -7,6 +8,40 @@ namespace ZB.MOM.WW.LocalDb.Internal;
|
||||
/// </summary>
|
||||
internal static class LocalDbSchema
|
||||
{
|
||||
/// <summary>
|
||||
/// The on-disk bookkeeping-schema version this build creates and upgrades to. Bumping it means
|
||||
/// adding a case to <see cref="Upgrade"/>; existing files are upgraded in place on open.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately NOT the same number as <c>SyncSession.LibSchemaVersion</c>, which is the WIRE
|
||||
/// protocol version and is compared fail-closed against the peer's. This one is local-file-only:
|
||||
/// two nodes may briefly sit on different file-schema versions during an upgrade and still sync,
|
||||
/// because nothing about the file layout crosses the wire.
|
||||
/// <list type="bullet">
|
||||
/// <item><description>
|
||||
/// <b>1</b> — the original layout, which also created <c>__localdb_oplog_hlc</c>.
|
||||
/// </description></item>
|
||||
/// <item><description>
|
||||
/// <b>2</b> — drops <c>__localdb_oplog_hlc</c>. That index (<c>table_name, pk_json, hlc</c>) was
|
||||
/// never read by any query: every oplog access path is by <c>seq</c> (the rowid) or a full
|
||||
/// <c>hlc</c>-predicate delete. It cost a b-tree insert on the CAPTURE path — i.e. inside every
|
||||
/// consumer write transaction — to serve nothing.
|
||||
/// </description></item>
|
||||
/// <item><description>
|
||||
/// <b>3</b> — adds <c>__localdb_snapshot_state</c>, the PER-TABLE snapshot-resync flag. Until
|
||||
/// v3 the only flag was <c>__localdb_peer_state.needs_snapshot</c>, one bit for the whole
|
||||
/// database, so baselining a single table re-streamed EVERY registered table in full — in both
|
||||
/// directions at once on a pair enabling replication late. The database-wide bit survives with
|
||||
/// its original meaning (the oplog-horizon and cap-prune paths genuinely invalidate every
|
||||
/// table); the new table narrows the baseline case to the tables that actually need it. The
|
||||
/// transition itself is a version stamp only — <see cref="Ddl"/> creates the table with
|
||||
/// <c>IF NOT EXISTS</c> on every open, so an older file gains it whatever its stamp says, and
|
||||
/// an empty flag table means exactly what it meant before: nothing per-table is owed.
|
||||
/// </description></item>
|
||||
/// </list>
|
||||
/// </remarks>
|
||||
internal const long CurrentSchemaVersion = 3;
|
||||
|
||||
private const string Ddl = """
|
||||
CREATE TABLE IF NOT EXISTS __localdb_meta (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
@@ -25,7 +60,6 @@ internal static class LocalDbSchema
|
||||
node_id TEXT NOT NULL,
|
||||
is_tombstone INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS __localdb_oplog_hlc ON __localdb_oplog(table_name, pk_json, hlc);
|
||||
CREATE TABLE IF NOT EXISTS __localdb_peer_state (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
peer_node_id TEXT NULL,
|
||||
@@ -50,6 +84,14 @@ internal static class LocalDbSchema
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
applying INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
-- Per-table snapshot-resync flags (schema v3). A row exists only while that table owes the
|
||||
-- peer a snapshot; the whole-database __localdb_peer_state.needs_snapshot bit still means
|
||||
-- "every registered table" and is what the cap/horizon paths set.
|
||||
-- WITHOUT ROWID: table_name IS the only access path (read as a set, cleared by name).
|
||||
CREATE TABLE IF NOT EXISTS __localdb_snapshot_state (
|
||||
table_name TEXT NOT NULL PRIMARY KEY,
|
||||
needs_snapshot INTEGER NOT NULL DEFAULT 0
|
||||
) WITHOUT ROWID;
|
||||
CREATE TABLE IF NOT EXISTS __localdb_dead_letter (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
received_utc TEXT NOT NULL,
|
||||
@@ -80,14 +122,67 @@ internal static class LocalDbSchema
|
||||
meta.Transaction = transaction;
|
||||
meta.CommandText =
|
||||
"INSERT INTO __localdb_meta (id, node_id, schema_version) " +
|
||||
"SELECT 1, $nodeId, 1 WHERE NOT EXISTS (SELECT 1 FROM __localdb_meta)";
|
||||
"SELECT 1, $nodeId, $version WHERE NOT EXISTS (SELECT 1 FROM __localdb_meta)";
|
||||
meta.Parameters.AddWithValue("$nodeId", Guid.NewGuid().ToString("D"));
|
||||
meta.Parameters.AddWithValue("$version", CurrentSchemaVersion);
|
||||
meta.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
Upgrade(connection, transaction);
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Brings a file created by an OLDER build up to <see cref="CurrentSchemaVersion"/>, in the
|
||||
/// caller's transaction. A file already at the current version is untouched.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Each step is written to be safe to re-run (IF EXISTS / IF NOT EXISTS), so a crash between the
|
||||
/// DDL and the version stamp costs nothing but a repeat on the next open. Steps run in ascending
|
||||
/// order and the stamp is written once at the end, inside the same transaction as the DDL — a
|
||||
/// file is therefore never observed at the new version with an unapplied step.
|
||||
/// </remarks>
|
||||
private static void Upgrade(SqliteConnection connection, SqliteTransaction transaction)
|
||||
{
|
||||
long current;
|
||||
using (var read = connection.CreateCommand())
|
||||
{
|
||||
read.Transaction = transaction;
|
||||
read.CommandText = "SELECT schema_version FROM __localdb_meta WHERE id = 1";
|
||||
current = Convert.ToInt64(read.ExecuteScalar(), CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
if (current >= CurrentSchemaVersion)
|
||||
return;
|
||||
|
||||
var steps = new List<string>();
|
||||
|
||||
// 1 -> 2: drop the write-amplifying, never-read oplog index (see CurrentSchemaVersion).
|
||||
if (current < 2)
|
||||
steps.Add("DROP INDEX IF EXISTS __localdb_oplog_hlc;");
|
||||
|
||||
// 2 -> 3: __localdb_snapshot_state. No step: the shared Ddl above already ran
|
||||
// CREATE TABLE IF NOT EXISTS for it in this same transaction, and an empty flag table is
|
||||
// the correct v3 state for a file that has never baselined a table. Repeating the DDL here
|
||||
// would be a second source of truth for the same layout. The stamp still moves, because
|
||||
// schema_version records what the file's layout IS, not which statements produced it.
|
||||
|
||||
if (steps.Count > 0)
|
||||
{
|
||||
using var ddl = connection.CreateCommand();
|
||||
ddl.Transaction = transaction;
|
||||
ddl.CommandText = string.Concat(steps);
|
||||
ddl.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
using var stamp = connection.CreateCommand();
|
||||
stamp.Transaction = transaction;
|
||||
stamp.CommandText = "UPDATE __localdb_meta SET schema_version = $version WHERE id = 1";
|
||||
stamp.Parameters.AddWithValue("$version", CurrentSchemaVersion);
|
||||
stamp.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
internal static (string NodeId, long HlcHighWater) ReadMeta(SqliteConnection connection)
|
||||
{
|
||||
using var cmd = connection.CreateCommand();
|
||||
@@ -99,10 +194,22 @@ internal static class LocalDbSchema
|
||||
return (reader.GetString(0), reader.GetInt64(1));
|
||||
}
|
||||
|
||||
internal static void FlushHlcHighWater(SqliteConnection connection, long value)
|
||||
/// <summary>
|
||||
/// Writes the clock's durable crash anchor. <paramref name="transaction"/> must be supplied when
|
||||
/// the connection has one open (SQLite commands are rejected otherwise), which is what lets a
|
||||
/// caller flush the anchor atomically with a change that would otherwise erase the clock's other
|
||||
/// recovery evidence — see <see cref="ILocalDb.DeregisterReplicated"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// MAX-guarded rather than an assignment: every writer of this column is flushing a monotonic
|
||||
/// clock, so a lower value can only be a stale one, and the anchor exists precisely to never
|
||||
/// regress.
|
||||
/// </remarks>
|
||||
internal static void FlushHlcHighWater(SqliteConnection connection, long value, SqliteTransaction? transaction = null)
|
||||
{
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = "UPDATE __localdb_meta SET hlc_high_water = $value WHERE id = 1";
|
||||
cmd.Transaction = transaction;
|
||||
cmd.CommandText = "UPDATE __localdb_meta SET hlc_high_water = MAX(hlc_high_water, $value) WHERE id = 1";
|
||||
cmd.Parameters.AddWithValue("$value", value);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
@@ -138,7 +138,15 @@ internal sealed class SqliteLocalDb : ILocalDb, IDisposable
|
||||
return await ReadAllAsync(cmd, map, ct);
|
||||
}
|
||||
|
||||
public async Task<ILocalDbTransaction> BeginTransactionAsync(CancellationToken ct = default)
|
||||
public async Task<ILocalDbTransaction> BeginTransactionAsync(CancellationToken ct = default) =>
|
||||
await BeginPreparedTransactionAsync(ct);
|
||||
|
||||
/// <summary>
|
||||
/// As <see cref="BeginTransactionAsync"/>, but typed so internal callers reach the underlying
|
||||
/// connection and transaction — which is what lets a hot loop hold <see cref="SqliteCommand"/>s
|
||||
/// across many rows and rebind them, instead of re-preparing statements per statement per row.
|
||||
/// </summary>
|
||||
internal async Task<Transaction> BeginPreparedTransactionAsync(CancellationToken ct = default)
|
||||
{
|
||||
var conn = await CreateConnectionAsync(ct);
|
||||
try
|
||||
@@ -155,8 +163,11 @@ internal sealed class SqliteLocalDb : ILocalDb, IDisposable
|
||||
|
||||
public IReadOnlyDictionary<string, ReplicatedTable> ReplicatedTables => _replicatedTables;
|
||||
|
||||
public ReplicatedTable RegisterReplicated(string tableName) =>
|
||||
RegisterReplicated(tableName, baselineExistingRows: false);
|
||||
|
||||
// Thread-safe: all master-connection work runs under _masterLock.
|
||||
public ReplicatedTable RegisterReplicated(string tableName)
|
||||
public ReplicatedTable RegisterReplicated(string tableName, bool baselineExistingRows)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tableName) || !TableNameRegex.IsMatch(tableName))
|
||||
throw new LocalDbRegistrationException(
|
||||
@@ -188,6 +199,8 @@ internal sealed class SqliteLocalDb : ILocalDb, IDisposable
|
||||
throw new LocalDbRegistrationException(
|
||||
$"Cannot register table '{tableName}': replication requires an explicit primary key.");
|
||||
|
||||
EnsureNoRowMutatingDeleteAction(tableName);
|
||||
|
||||
var allColumns = columns.Select(c => (c.Name, c.Type)).ToList();
|
||||
var digest = ReplicatedTable.ComputeDigest(tableName, pkColumns, allColumns);
|
||||
var table = new ReplicatedTable(tableName, pkColumns, allColumns, digest);
|
||||
@@ -201,6 +214,10 @@ internal sealed class SqliteLocalDb : ILocalDb, IDisposable
|
||||
cmd.CommandText = script;
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
if (baselineExistingRows)
|
||||
BaselineExistingRows(tx, table);
|
||||
|
||||
tx.Commit();
|
||||
}
|
||||
|
||||
@@ -209,6 +226,214 @@ internal sealed class SqliteLocalDb : ILocalDb, IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
// Thread-safe: all master-connection work runs under _masterLock.
|
||||
public bool DeregisterReplicated(string tableName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tableName) || !TableNameRegex.IsMatch(tableName))
|
||||
throw new LocalDbRegistrationException(
|
||||
$"Cannot deregister table '{tableName}': name must be a plain identifier matching {TableNameRegex}.");
|
||||
|
||||
lock (_masterLock)
|
||||
{
|
||||
using var tx = _master.BeginTransaction();
|
||||
|
||||
// Counted BEFORE the drop: DROP TRIGGER reports no row count, and "did anything change?"
|
||||
// is the caller's cue to log a one-time cleanup rather than a no-op every boot.
|
||||
long triggersPresent;
|
||||
using (var probe = _master.CreateCommand())
|
||||
{
|
||||
probe.Transaction = tx;
|
||||
probe.CommandText =
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger' AND name IN ($ai, $au, $ad)";
|
||||
foreach (var suffix in TriggerSqlGenerator.Suffixes)
|
||||
probe.Parameters.AddWithValue("$" + suffix, TriggerSqlGenerator.TriggerName(tableName, suffix));
|
||||
triggersPresent = Convert.ToInt64(probe.ExecuteScalar(), CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
using (var drop = _master.CreateCommand())
|
||||
{
|
||||
drop.Transaction = tx;
|
||||
drop.CommandText = TriggerSqlGenerator.BuildUninstallScript(tableName);
|
||||
drop.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
// Flushed BEFORE the prune below, in the SAME transaction, because that prune destroys
|
||||
// the clock's crash evidence. Recovery on open takes a MAX over hlc_high_water, the
|
||||
// oplog's HLCs, the row-version ledger's HLCs and last_seen_hlc — and hlc_high_water is
|
||||
// otherwise written only by Dispose. Deregistering a table deletes that table's oplog
|
||||
// AND ledger rows, so on a file whose registered tables are all deregistered, an
|
||||
// UNGRACEFUL exit (crash, SIGKILL, power loss — no Dispose) afterwards leaves nothing
|
||||
// above zero to recover from: the reopened clock would restart at wall-clock time and
|
||||
// could re-issue HLCs below stamps peers have already observed, which LWW reads as
|
||||
// "older" and discards. One row-level UPDATE on an administrative one-shot buys the
|
||||
// anchor back; it is a separate command from the prune so it cannot inflate prunedRows.
|
||||
LocalDbSchema.FlushHlcHighWater(_master, _clock.Current, tx);
|
||||
|
||||
// Full scan of the oplog by table_name — the index that would have served it is exactly
|
||||
// the one schema v2 drops, and paying a write on every capture to speed up a one-shot
|
||||
// administrative prune is the wrong trade.
|
||||
int prunedRows;
|
||||
using (var prune = _master.CreateCommand())
|
||||
{
|
||||
prune.Transaction = tx;
|
||||
prune.CommandText =
|
||||
"DELETE FROM __localdb_oplog WHERE table_name = $t;" +
|
||||
"DELETE FROM __localdb_row_version WHERE table_name = $t;" +
|
||||
// A pending per-table snapshot flag for a table nothing streams any more would
|
||||
// never clear — the streamer skips unregistered tables, so the flag would sit
|
||||
// in the file forever, and re-registering later would find a stale "already
|
||||
// owed" it did not ask for. Dropped with the ledger it refers to.
|
||||
"DELETE FROM __localdb_snapshot_state WHERE table_name = $t;";
|
||||
prune.Parameters.AddWithValue("$t", tableName);
|
||||
prunedRows = prune.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
tx.Commit();
|
||||
|
||||
_replicatedTables.TryRemove(tableName, out _);
|
||||
return triggersPresent > 0 || prunedRows > 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Refuses to register <paramref name="tableName"/> when another table references it with an
|
||||
/// <c>ON DELETE</c> action that MUTATES the referencing rows (<c>CASCADE</c>, <c>SET NULL</c>,
|
||||
/// <c>SET DEFAULT</c>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why this is a replication correctness rule and not a schema opinion.</b> Applying a peer's
|
||||
/// row runs <c>INSERT OR REPLACE</c>, and REPLACE resolves a primary-key conflict by DELETING the
|
||||
/// existing row first. Connections run with <c>PRAGMA foreign_keys=ON</c>, so that implicit delete
|
||||
/// fires the referential action on every child table — while the apply path holds the capture
|
||||
/// guard raised, precisely so its own writes are not re-captured. The child rows are therefore
|
||||
/// deleted (or nulled) locally and produce NO oplog entry: the peer never hears about them, keeps
|
||||
/// them, and the pair is durably divergent with nothing anywhere reporting a fault.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Refusing at registration is the only place the problem is visible. At apply time the damage is
|
||||
/// already done and indistinguishable from a normal write, and no runtime signal exists to detect
|
||||
/// it. <b>NO ACTION / RESTRICT are deliberately allowed</b>: they mutate no child row — the
|
||||
/// implicit delete simply fails, which the applier surfaces as a dead-lettered entry. Loud, not
|
||||
/// silent, so there is nothing to guard.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The check is registration-time, so a cascading child table CREATED LATER is not caught (SQLite
|
||||
/// has no DDL hook to catch it on). Registration normally runs at startup after schema setup,
|
||||
/// which is where the constraint belongs.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private void EnsureNoRowMutatingDeleteAction(string tableName)
|
||||
{
|
||||
var candidates = new List<string>();
|
||||
using (var cmd = _master.CreateCommand())
|
||||
{
|
||||
// Every user table is a candidate CHILD; the internal sqlite_* tables carry no
|
||||
// foreign keys. The registered table itself is included: a self-referencing cascade
|
||||
// is the same hazard.
|
||||
cmd.CommandText = "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'";
|
||||
using var reader = cmd.ExecuteReader();
|
||||
while (reader.Read())
|
||||
candidates.Add(reader.GetString(0));
|
||||
}
|
||||
|
||||
foreach (var child in candidates)
|
||||
{
|
||||
using var cmd = _master.CreateCommand();
|
||||
cmd.CommandText = $"PRAGMA foreign_key_list({TriggerSqlGenerator.Ident(child)})";
|
||||
using var reader = cmd.ExecuteReader();
|
||||
while (reader.Read())
|
||||
{
|
||||
// foreign_key_list cols: 0=id, 1=seq, 2=table (the PARENT), 3=from, 4=to,
|
||||
// 5=on_update, 6=on_delete, 7=match.
|
||||
var parent = reader.IsDBNull(2) ? "" : reader.GetString(2);
|
||||
// SQLite identifiers are case-insensitive, and the REFERENCES clause need not
|
||||
// match the CREATE TABLE spelling.
|
||||
if (!string.Equals(parent, tableName, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
var onDelete = reader.IsDBNull(6) ? "" : reader.GetString(6);
|
||||
if (!RowMutatingDeleteActions.Contains(onDelete))
|
||||
continue;
|
||||
|
||||
throw new LocalDbRegistrationException(
|
||||
$"Cannot register table '{tableName}': table '{child}' has a foreign key referencing it " +
|
||||
$"with ON DELETE {onDelete}. Replicated tables must not be cascade-delete foreign-key " +
|
||||
"parents: applying a remote row uses INSERT OR REPLACE, whose implicit DELETE fires the " +
|
||||
"referential action while change capture is suppressed — the child rows it touches are " +
|
||||
"never captured, so the peer never learns of them and the two nodes diverge silently. " +
|
||||
$"Drop the referential action on '{child}' (ON DELETE NO ACTION, removing children " +
|
||||
"explicitly instead), or do not replicate this table.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The <c>ON DELETE</c> actions that mutate referencing rows, as SQLite spells them in
|
||||
/// <c>PRAGMA foreign_key_list</c>. NO ACTION / RESTRICT are absent on purpose — see
|
||||
/// <see cref="EnsureNoRowMutatingDeleteAction"/>.
|
||||
/// </summary>
|
||||
private static readonly HashSet<string> RowMutatingDeleteActions =
|
||||
new(["CASCADE", "SET NULL", "SET DEFAULT"], StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Seeds <c>__localdb_row_version</c> at the baseline floor for every row of
|
||||
/// <paramref name="table"/> that has no ledger entry, and flags a snapshot resync if it seeded
|
||||
/// anything. Runs in the registration transaction so triggers and baseline commit together.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The floor is HLC <c>0</c> — see <see cref="ILocalDb.RegisterReplicated(string, bool)"/> for
|
||||
/// why that is the only LWW-safe choice.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The <c>needs_snapshot</c> flag is what makes this actually deliver.</b> Seeding the ledger
|
||||
/// alone changes nothing on the wire: a node whose oplog starts at seq 1 presents no gap at the
|
||||
/// handshake, so the peer is told it needs no snapshot and only ever receives post-registration
|
||||
/// deltas. Flagging the resync is what turns the seeded ledger into rows the peer receives.
|
||||
/// Deliberately NOT done by writing oplog entries instead: that would dump the entire table into
|
||||
/// the oplog on the first boot after opting in, likely blowing <c>MaxOplogRows</c> and forcing
|
||||
/// the same snapshot anyway, via a cap-prune.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The flag is PER-TABLE</b> (<c>__localdb_snapshot_state</c>, schema v3): baselining one
|
||||
/// table asks for a snapshot of that table alone, not of every registered table. The
|
||||
/// database-wide <c>__localdb_peer_state.needs_snapshot</c> bit is untouched here and keeps its
|
||||
/// original meaning — the oplog-horizon and cap-prune paths invalidate every table's delta
|
||||
/// continuity at once, and only those paths set it. A peer too old to understand a scoped
|
||||
/// snapshot is served a full one instead; the negotiation lives in the replication package.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private void BaselineExistingRows(SqliteTransaction tx, ReplicatedTable table)
|
||||
{
|
||||
int seeded;
|
||||
using (var cmd = _master.CreateCommand())
|
||||
{
|
||||
cmd.Transaction = tx;
|
||||
// The trailing WHERE 1 disambiguates the upsert parse on a SELECT source — the same
|
||||
// reason TriggerSqlGenerator's row-version statements carry one.
|
||||
cmd.CommandText =
|
||||
"INSERT INTO __localdb_row_version (table_name, pk_json, hlc, node_id, is_tombstone, tombstone_utc) " +
|
||||
$"SELECT $t, {TriggerSqlGenerator.JsonObject("t", table.PkColumns)}, 0, $node, 0, NULL " +
|
||||
$"FROM {TriggerSqlGenerator.Ident(table.Name)} t WHERE 1 " +
|
||||
"ON CONFLICT(table_name, pk_json) DO NOTHING";
|
||||
cmd.Parameters.AddWithValue("$t", table.Name);
|
||||
cmd.Parameters.AddWithValue("$node", _nodeId);
|
||||
seeded = cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
if (seeded == 0)
|
||||
return;
|
||||
|
||||
using var flag = _master.CreateCommand();
|
||||
flag.Transaction = tx;
|
||||
flag.CommandText =
|
||||
"INSERT INTO __localdb_snapshot_state (table_name, needs_snapshot) VALUES ($t, 1) " +
|
||||
"ON CONFLICT(table_name) DO UPDATE SET needs_snapshot = 1";
|
||||
flag.Parameters.AddWithValue("$t", table.Name);
|
||||
flag.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
@@ -329,8 +554,14 @@ internal sealed class SqliteLocalDb : ILocalDb, IDisposable
|
||||
return Convert.ToInt64(cmd.ExecuteScalar(), CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private sealed class Transaction(SqliteConnection connection, SqliteTransaction transaction) : ILocalDbTransaction
|
||||
internal sealed class Transaction(SqliteConnection connection, SqliteTransaction transaction) : ILocalDbTransaction
|
||||
{
|
||||
/// <summary>The pinned connection, for internal callers holding their own commands.</summary>
|
||||
internal SqliteConnection Connection => connection;
|
||||
|
||||
/// <summary>The underlying transaction, to assign to internally-held commands.</summary>
|
||||
internal SqliteTransaction Sqlite => transaction;
|
||||
|
||||
public async Task<int> ExecuteAsync(string sql, object? parameters = null, CancellationToken ct = default)
|
||||
{
|
||||
await using var cmd = connection.CreateCommand();
|
||||
|
||||
@@ -27,14 +27,16 @@ internal static class TriggerSqlGenerator
|
||||
"ON CONFLICT(table_name, pk_json) DO UPDATE SET hlc = excluded.hlc, node_id = excluded.node_id, " +
|
||||
"is_tombstone = excluded.is_tombstone, tombstone_utc = excluded.tombstone_utc";
|
||||
|
||||
/// <summary>The three trigger suffixes, in the order the install/uninstall scripts emit them.</summary>
|
||||
internal static readonly string[] Suffixes = ["ai", "au", "ad"];
|
||||
|
||||
public static string TriggerName(string table, string suffix) => $"__localdb_{table}_{suffix}";
|
||||
|
||||
/// <summary>DROP IF EXISTS + CREATE for all three triggers, ready to run in one transaction.</summary>
|
||||
public static string BuildInstallScript(ReplicatedTable table)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
foreach (var suffix in new[] { "ai", "au", "ad" })
|
||||
sb.Append("DROP TRIGGER IF EXISTS ").Append(Ident(TriggerName(table.Name, suffix))).Append(";\n");
|
||||
sb.Append(BuildUninstallScript(table.Name));
|
||||
|
||||
sb.Append(BuildInsert(table)).Append('\n');
|
||||
sb.Append(BuildUpdate(table)).Append('\n');
|
||||
@@ -42,6 +44,20 @@ internal static class TriggerSqlGenerator
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// DROP IF EXISTS for all three triggers of <paramref name="tableName"/>. Takes the bare name
|
||||
/// rather than a <see cref="ReplicatedTable"/> because uninstall must work for a table this
|
||||
/// process never registered — the whole point of the cleanup API is stale triggers left by an
|
||||
/// EARLIER build, whose column list (and therefore digest) is unavailable and irrelevant.
|
||||
/// </summary>
|
||||
public static string BuildUninstallScript(string tableName)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
foreach (var suffix in Suffixes)
|
||||
sb.Append("DROP TRIGGER IF EXISTS ").Append(Ident(TriggerName(tableName, suffix))).Append(";\n");
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static string BuildInsert(ReplicatedTable t) =>
|
||||
Wrap(t, "ai", "INSERT", NewRowCapture(t));
|
||||
|
||||
@@ -112,13 +128,16 @@ internal static class TriggerSqlGenerator
|
||||
}
|
||||
|
||||
// json_object('col', ALIAS."col", ...) — natively handles NULLs and SQLite scalar types.
|
||||
private static string JsonObject(string alias, IEnumerable<string> columns)
|
||||
// Internal rather than private: baseline seeding must build a pk_json identical to what the
|
||||
// capture triggers emit, character for character, or a seeded ledger key would never match the
|
||||
// key a later capture writes for the same row.
|
||||
internal static string JsonObject(string alias, IEnumerable<string> columns)
|
||||
{
|
||||
var args = columns.Select(c => $"{Literal(c)}, {alias}.{Ident(c)}");
|
||||
return $"json_object({string.Join(", ", args)})";
|
||||
}
|
||||
|
||||
private static string Ident(string name) => "\"" + name.Replace("\"", "\"\"") + "\"";
|
||||
internal static string Ident(string name) => "\"" + name.Replace("\"", "\"\"") + "\"";
|
||||
|
||||
private static string Literal(string value) => "'" + value.Replace("'", "''") + "'";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
using System.Threading.Channels;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.LocalDb.Contracts;
|
||||
using ZB.MOM.WW.LocalDb.Internal;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
using ZB.MOM.WW.LocalDb.Replication.Internal;
|
||||
|
||||
namespace ZB.MOM.WW.LocalDb.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Outbound batching by summed serialized BYTES (with the row count kept as a secondary cap), and
|
||||
/// the inbound coalescing that keeps one transaction + one ack per burst rather than per message.
|
||||
/// </summary>
|
||||
public sealed class BatchBudgetTests : IDisposable
|
||||
{
|
||||
private static readonly TimeSpan RunTimeout = TimeSpan.FromSeconds(15);
|
||||
|
||||
private readonly List<string> _paths = [];
|
||||
private readonly List<IDisposable> _disposables = [];
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var d in _disposables)
|
||||
d.Dispose();
|
||||
SqliteConnection.ClearAllPools();
|
||||
foreach (var p in _paths)
|
||||
{
|
||||
if (File.Exists(p)) File.Delete(p);
|
||||
if (File.Exists(p + "-wal")) File.Delete(p + "-wal");
|
||||
if (File.Exists(p + "-shm")) File.Delete(p + "-shm");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<SqliteLocalDb> NewOrdersDbAsync()
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
||||
_paths.Add(path);
|
||||
var db = new SqliteLocalDb(new LocalDbOptions { Path = path });
|
||||
_disposables.Add(db);
|
||||
await db.ExecuteAsync("CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT, qty INTEGER)");
|
||||
db.RegisterReplicated("orders");
|
||||
return db;
|
||||
}
|
||||
|
||||
private static (SyncDuplex Duplex, ChannelWriter<SyncMessage> ToSession, ChannelReader<SyncMessage> FromSession) ScriptedPeer()
|
||||
{
|
||||
var toSession = Channel.CreateUnbounded<SyncMessage>();
|
||||
var fromSession = Channel.CreateUnbounded<SyncMessage>();
|
||||
var duplex = new SyncDuplex
|
||||
{
|
||||
Send = (m, ct) => fromSession.Writer.WriteAsync(m, ct).AsTask(),
|
||||
Inbox = toSession.Reader,
|
||||
};
|
||||
return (duplex, toSession.Writer, fromSession.Reader);
|
||||
}
|
||||
|
||||
private static Handshake MatchingHandshake(SqliteLocalDb sessionDb, string peerNodeId)
|
||||
{
|
||||
var h = new Handshake { NodeId = peerNodeId, LibSchemaVersion = 1, LastAppliedRemoteSeq = 0 };
|
||||
foreach (var t in sessionDb.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal))
|
||||
h.Tables.Add(new TableDigest { TableName = t.Name, Digest = t.Digest });
|
||||
return h;
|
||||
}
|
||||
|
||||
private static async Task<SyncMessage> NextAsync(ChannelReader<SyncMessage> reader, CancellationToken ct)
|
||||
{
|
||||
await reader.WaitToReadAsync(ct);
|
||||
reader.TryRead(out var msg);
|
||||
return msg!;
|
||||
}
|
||||
|
||||
// Reads the next scripted-peer message while also observing the session task: a session that
|
||||
// faults would otherwise leave this wait to run out the whole test timeout, reporting an
|
||||
// opaque cancellation instead of the session's real exception.
|
||||
private static async Task<SyncMessage> NextAsync(Task run, ChannelReader<SyncMessage> reader, CancellationToken ct)
|
||||
{
|
||||
var next = NextAsync(reader, ct);
|
||||
if (ReferenceEquals(await Task.WhenAny(run, next), run))
|
||||
{
|
||||
await run;
|
||||
throw new InvalidOperationException("The sync session ended before the expected message arrived.");
|
||||
}
|
||||
return await next;
|
||||
}
|
||||
|
||||
private static async Task SwallowAsync(Task task)
|
||||
{
|
||||
try { await task; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
|
||||
// Queues a complete peer handshake AHEAD of the session's own, so anything queued after it is
|
||||
// guaranteed to be sitting in the inbox by the time the receive loop starts. That is what makes
|
||||
// the coalescing assertions deterministic rather than a race with the reader.
|
||||
private static async Task PreloadHandshakeAsync(
|
||||
SqliteLocalDb db, ChannelWriter<SyncMessage> toSession, CancellationToken ct)
|
||||
{
|
||||
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(db, "peer") }, ct);
|
||||
await toSession.WriteAsync(
|
||||
new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, ct);
|
||||
}
|
||||
|
||||
private static long BatchBytes(DeltaBatch batch) =>
|
||||
batch.Entries.Sum(e => 1L + Google.Protobuf.CodedOutputStream.ComputeMessageSize(e));
|
||||
|
||||
private static long BatchBytes(SnapshotBatch batch) =>
|
||||
batch.Rows.Sum(r => 1L + Google.Protobuf.CodedOutputStream.ComputeMessageSize(r));
|
||||
|
||||
private static SyncSession NewSession(SqliteLocalDb db, ReplicationOptions options)
|
||||
{
|
||||
var store = new OplogStore(db, options);
|
||||
var applier = new LwwApplier(db);
|
||||
return new SyncSession(db, store, applier, options, NullLogger.Instance);
|
||||
}
|
||||
|
||||
private static OplogEntry ProtoEntry(long seq, long id, long hlc) =>
|
||||
new()
|
||||
{
|
||||
Seq = seq,
|
||||
TableName = "orders",
|
||||
PkJson = $"{{\"id\":{id}}}",
|
||||
RowJson = $"{{\"id\":{id},\"sku\":\"S\",\"qty\":1}}",
|
||||
Hlc = hlc,
|
||||
NodeId = "peer",
|
||||
IsTombstone = false,
|
||||
};
|
||||
|
||||
// ---- Outbound: delta pump --------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Pump_SplitsOneReadIntoSeveralMessages_EachWithinTheByteBudget()
|
||||
{
|
||||
const int budget = 4_000;
|
||||
var db = await NewOrdersDbAsync();
|
||||
var wide = new string('x', 900);
|
||||
for (var i = 1; i <= 20; i++)
|
||||
await db.ExecuteAsync(
|
||||
"INSERT INTO orders (id, sku, qty) VALUES (@id, @sku, @id)", new { id = i, sku = wide });
|
||||
|
||||
// Row count alone would put all 20 wide rows (~18 KB) in ONE message; the byte budget is
|
||||
// what actually bounds the message the transport has to carry.
|
||||
var options = new ReplicationOptions
|
||||
{
|
||||
FlushInterval = TimeSpan.FromMilliseconds(20), MaxBatchSize = 500, MaxBatchBytes = budget,
|
||||
};
|
||||
var session = NewSession(db, options);
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = session.RunAsync(duplex, cts.Token);
|
||||
await PreloadHandshakeAsync(db, toSession, cts.Token);
|
||||
|
||||
var seen = new List<long>();
|
||||
while (seen.Count < 20)
|
||||
{
|
||||
var msg = await NextAsync(run, fromSession, cts.Token);
|
||||
if (msg.MsgCase != SyncMessage.MsgOneofCase.DeltaBatch)
|
||||
continue;
|
||||
Assert.True(BatchBytes(msg.DeltaBatch) <= budget,
|
||||
$"a {BatchBytes(msg.DeltaBatch)}-byte message exceeded the {budget}-byte budget");
|
||||
Assert.True(msg.DeltaBatch.Entries.Count > 1, "the budget should still fit several of these rows");
|
||||
seen.AddRange(msg.DeltaBatch.Entries.Select(e => e.Seq));
|
||||
}
|
||||
|
||||
// Split, not dropped or duplicated: every captured seq travelled exactly once, in order.
|
||||
Assert.Equal(20, seen.Count);
|
||||
Assert.Equal(seen.OrderBy(x => x).ToList(), seen);
|
||||
Assert.Equal(20, seen.Distinct().Count());
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pump_RowLargerThanTheBudget_IsSentAloneRatherThanStallingTheStream()
|
||||
{
|
||||
var db = await NewOrdersDbAsync();
|
||||
await db.ExecuteAsync(
|
||||
"INSERT INTO orders (id, sku, qty) VALUES (1, @sku, 1)", new { sku = new string('x', 5_000) });
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (2, 'small', 2)");
|
||||
|
||||
var options = new ReplicationOptions
|
||||
{
|
||||
FlushInterval = TimeSpan.FromMilliseconds(20), MaxBatchSize = 500, MaxBatchBytes = 100,
|
||||
};
|
||||
var session = NewSession(db, options);
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = session.RunAsync(duplex, cts.Token);
|
||||
await PreloadHandshakeAsync(db, toSession, cts.Token);
|
||||
|
||||
var batches = new List<DeltaBatch>();
|
||||
while (batches.Sum(b => b.Entries.Count) < 2)
|
||||
{
|
||||
var msg = await NextAsync(run, fromSession, cts.Token);
|
||||
if (msg.MsgCase == SyncMessage.MsgOneofCase.DeltaBatch)
|
||||
batches.Add(msg.DeltaBatch);
|
||||
}
|
||||
|
||||
// The oversized row goes out on its own — refusing it would wedge the stream on one row
|
||||
// forever, and everything behind it with it.
|
||||
Assert.Equal(2, batches.Count);
|
||||
Assert.Single(batches[0].Entries);
|
||||
Assert.Equal(1, batches[0].Entries[0].Seq);
|
||||
Assert.True(BatchBytes(batches[0]) > options.MaxBatchBytes);
|
||||
Assert.Single(batches[1].Entries);
|
||||
Assert.Equal(2, batches[1].Entries[0].Seq);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pump_RowCountRemainsASecondaryCap()
|
||||
{
|
||||
var db = await NewOrdersDbAsync();
|
||||
for (var i = 1; i <= 12; i++)
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'S', @id)", new { id = i });
|
||||
|
||||
// Tiny rows, enormous byte budget: only the row cap can be doing the splitting here.
|
||||
var options = new ReplicationOptions
|
||||
{
|
||||
FlushInterval = TimeSpan.FromMilliseconds(20), MaxBatchSize = 5, MaxBatchBytes = 8 * 1024 * 1024,
|
||||
};
|
||||
var session = NewSession(db, options);
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = session.RunAsync(duplex, cts.Token);
|
||||
await PreloadHandshakeAsync(db, toSession, cts.Token);
|
||||
|
||||
var seen = 0;
|
||||
while (seen < 12)
|
||||
{
|
||||
var msg = await NextAsync(run, fromSession, cts.Token);
|
||||
if (msg.MsgCase != SyncMessage.MsgOneofCase.DeltaBatch)
|
||||
continue;
|
||||
Assert.True(msg.DeltaBatch.Entries.Count <= 5);
|
||||
seen += msg.DeltaBatch.Entries.Count;
|
||||
}
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
// ---- Outbound: snapshot --------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Snapshot_SplitsPagesIntoMessagesWithinTheByteBudget()
|
||||
{
|
||||
const int budget = 4_000;
|
||||
var db = await NewOrdersDbAsync();
|
||||
var wide = new string('x', 900);
|
||||
for (var i = 1; i <= 20; i++)
|
||||
await db.ExecuteAsync(
|
||||
"INSERT INTO orders (id, sku, qty) VALUES (@id, @sku, @id)", new { id = i, sku = wide });
|
||||
|
||||
// A snapshot is the easiest way to blow a message limit: it ships whole rows, all of them,
|
||||
// back to back — so its page size has to be bounded in bytes too, not just in rows.
|
||||
var options = new ReplicationOptions { MaxBatchSize = 500, MaxBatchBytes = budget };
|
||||
var store = new OplogStore(db, options);
|
||||
var streamer = new SnapshotStreamer(db, store, options, NullLogger.Instance);
|
||||
|
||||
var sent = new List<SyncMessage>();
|
||||
await streamer.SendAsync(SnapshotScope.Full, (m, _) => { sent.Add(m); return Task.CompletedTask; }, default);
|
||||
|
||||
var batches = sent.Where(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBatch)
|
||||
.Select(m => m.SnapshotBatch).ToList();
|
||||
Assert.True(batches.Count > 1, "20 wide rows must not have travelled as one message");
|
||||
Assert.All(batches, b => Assert.True(BatchBytes(b) <= budget,
|
||||
$"a {BatchBytes(b)}-byte snapshot message exceeded the {budget}-byte budget"));
|
||||
|
||||
// Bracketed and complete: every row, exactly once.
|
||||
Assert.Equal(SyncMessage.MsgOneofCase.SnapshotBegin, sent[0].MsgCase);
|
||||
Assert.Equal(SyncMessage.MsgOneofCase.SnapshotComplete, sent[^1].MsgCase);
|
||||
var pks = batches.SelectMany(b => b.Rows).Select(r => r.PkJson).ToList();
|
||||
Assert.Equal(20, pks.Count);
|
||||
Assert.Equal(20, pks.Distinct().Count());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Snapshot_RowLargerThanTheBudget_IsSentAlone()
|
||||
{
|
||||
var db = await NewOrdersDbAsync();
|
||||
await db.ExecuteAsync(
|
||||
"INSERT INTO orders (id, sku, qty) VALUES (1, @sku, 1)", new { sku = new string('x', 5_000) });
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (2, 'small', 2)");
|
||||
|
||||
var options = new ReplicationOptions { MaxBatchSize = 500, MaxBatchBytes = 100 };
|
||||
var store = new OplogStore(db, options);
|
||||
var streamer = new SnapshotStreamer(db, store, options, NullLogger.Instance);
|
||||
|
||||
var sent = new List<SyncMessage>();
|
||||
await streamer.SendAsync(SnapshotScope.Full, (m, _) => { sent.Add(m); return Task.CompletedTask; }, default);
|
||||
|
||||
var batches = sent.Where(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBatch)
|
||||
.Select(m => m.SnapshotBatch).ToList();
|
||||
Assert.Equal(2, batches.Count);
|
||||
Assert.All(batches, b => Assert.Single(b.Rows));
|
||||
Assert.Equal("{\"id\":1}", batches[0].Rows[0].PkJson);
|
||||
Assert.Equal("{\"id\":2}", batches[1].Rows[0].PkJson);
|
||||
}
|
||||
|
||||
// ---- Inbound: coalescing -------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Receive_CoalescesQueuedDeltaBatches_IntoOneAck()
|
||||
{
|
||||
var db = await NewOrdersDbAsync();
|
||||
var options = new ReplicationOptions { FlushInterval = TimeSpan.FromSeconds(30) };
|
||||
var session = NewSession(db, options);
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = session.RunAsync(duplex, cts.Token);
|
||||
|
||||
// Queued ahead of the receive loop, so all three are demonstrably in the inbox together.
|
||||
await PreloadHandshakeAsync(db, toSession, cts.Token);
|
||||
for (var i = 1; i <= 3; i++)
|
||||
{
|
||||
var batch = new DeltaBatch();
|
||||
batch.Entries.Add(ProtoEntry(i, i, 1_000_000 + i));
|
||||
await toSession.WriteAsync(new SyncMessage { DeltaBatch = batch }, cts.Token);
|
||||
}
|
||||
|
||||
SyncMessage ack;
|
||||
do { ack = await NextAsync(run, fromSession, cts.Token); }
|
||||
while (ack.MsgCase != SyncMessage.MsgOneofCase.DeltaAck);
|
||||
|
||||
// The discriminator: uncoalesced, the FIRST ack could only cover seq 1.
|
||||
Assert.Equal(3, ack.DeltaAck.AppliedThruSeq);
|
||||
|
||||
var rows = await db.QueryAsync("SELECT COUNT(*) FROM orders", r => r.GetInt64(0));
|
||||
Assert.Equal(3, rows[0]);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Receive_DeltaCoalescing_DoesNotSwallowTheMessageThatEndsTheRun()
|
||||
{
|
||||
// The risk the one-message lookahead exists to handle: coalescing reads ahead, and whatever
|
||||
// it reads that is NOT a delta must still be processed, not dropped on the floor.
|
||||
var db = await NewOrdersDbAsync();
|
||||
var options = new ReplicationOptions { FlushInterval = TimeSpan.FromSeconds(30) };
|
||||
var session = NewSession(db, options);
|
||||
var captured = new CapturingSnapshotApplier();
|
||||
session.SnapshotApplier = captured;
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await PreloadHandshakeAsync(db, toSession, cts.Token);
|
||||
var batch = new DeltaBatch();
|
||||
batch.Entries.Add(ProtoEntry(1, 1, 1_000_001));
|
||||
await toSession.WriteAsync(new SyncMessage { DeltaBatch = batch }, cts.Token);
|
||||
await toSession.WriteAsync(
|
||||
new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 77 } }, cts.Token);
|
||||
|
||||
SyncMessage ack;
|
||||
do { ack = await NextAsync(run, fromSession, cts.Token); }
|
||||
while (ack.MsgCase != SyncMessage.MsgOneofCase.DeltaAck);
|
||||
Assert.Equal(1, ack.DeltaAck.AppliedThruSeq);
|
||||
|
||||
// The SnapshotBegin that terminated the coalescing run still reached its handler.
|
||||
await WaitForAsync(() => captured.BeginAsOfSeq == 77, RunTimeout);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Receive_CoalescesQueuedAcks_ToTheHighestWatermark_WithoutLosingTheNextMessage()
|
||||
{
|
||||
var db = await NewOrdersDbAsync();
|
||||
for (var i = 1; i <= 9; i++)
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'S', @id)", new { id = i });
|
||||
|
||||
var options = new ReplicationOptions { FlushInterval = TimeSpan.FromSeconds(30) };
|
||||
var session = NewSession(db, options);
|
||||
var captured = new CapturingSnapshotApplier();
|
||||
session.SnapshotApplier = captured;
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await PreloadHandshakeAsync(db, toSession, cts.Token);
|
||||
// Out of order on purpose: collapsing acks must take the MAX, never the last.
|
||||
foreach (var seq in new long[] { 4, 9, 2 })
|
||||
await toSession.WriteAsync(
|
||||
new SyncMessage { DeltaAck = new DeltaAck { AppliedThruSeq = seq } }, cts.Token);
|
||||
await toSession.WriteAsync(
|
||||
new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 55 } }, cts.Token);
|
||||
|
||||
await WaitForAsync(async () =>
|
||||
{
|
||||
var acked = await db.QueryAsync(
|
||||
"SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1", r => r.GetInt64(0));
|
||||
return acked[0] == 9;
|
||||
}, RunTimeout);
|
||||
|
||||
Assert.Equal(9, session.PeerAckedSeq);
|
||||
await WaitForAsync(() => captured.BeginAsOfSeq == 55, RunTimeout);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
private sealed class CapturingSnapshotApplier : ISnapshotApplier
|
||||
{
|
||||
private long _beginAsOfSeq = -1;
|
||||
|
||||
public long BeginAsOfSeq => Interlocked.Read(ref _beginAsOfSeq);
|
||||
|
||||
public Task OnBeginAsync(SnapshotBegin begin, CancellationToken ct)
|
||||
{
|
||||
Interlocked.Exchange(ref _beginAsOfSeq, begin.AsOfSeq);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task OnBatchAsync(SnapshotBatch batch, CancellationToken ct) => Task.CompletedTask;
|
||||
|
||||
public Task OnCompleteAsync(SnapshotComplete complete, CancellationToken ct) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static Task WaitForAsync(Func<bool> predicate, TimeSpan timeout) =>
|
||||
WaitForAsync(() => Task.FromResult(predicate()), timeout);
|
||||
|
||||
private static async Task WaitForAsync(Func<Task<bool>> predicate, TimeSpan timeout)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (await predicate()) return;
|
||||
await Task.Delay(20);
|
||||
}
|
||||
throw new TimeoutException("Condition not reached within " + timeout);
|
||||
}
|
||||
}
|
||||
@@ -70,8 +70,12 @@ public sealed class GrpcAdapterTests : IAsyncLifetime
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecondConcurrentStream_Rejected()
|
||||
public async Task SecondConcurrentStream_DisplacesTheIncumbent()
|
||||
{
|
||||
// Newest wins (0.3.2). The guard serializes sessions; it was never a fence — the auth
|
||||
// interceptor is, and an authenticated same-key dialler IS the peer. Refusing the newcomer
|
||||
// bought nothing and cost the pair its only route back from a half-open link, where the
|
||||
// incumbent handler can hold the guard indefinitely on a dead stream.
|
||||
var serverHost = await BuildServerAsync(NewDbPath(), apiKey: null, sink: null);
|
||||
var serverDb = serverHost.Services.GetRequiredService<ILocalDb>();
|
||||
var handler = serverHost.GetTestServer().CreateHandler();
|
||||
@@ -85,11 +89,15 @@ public sealed class GrpcAdapterTests : IAsyncLifetime
|
||||
|
||||
// Convergence proves the first stream's server handler is active and holding the single-stream guard.
|
||||
await WaitForAsync(async () => (await ReadOrders(serverDb)).Count == 1, RunTimeout);
|
||||
var attemptsBefore = bg.ConnectionAttempts;
|
||||
|
||||
var client = new LocalDbSync.LocalDbSyncClient(ChannelOver(handler));
|
||||
using var call2 = client.Sync(cancellationToken: cts.Token);
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() => call2.ResponseStream.MoveNext(cts.Token));
|
||||
Assert.Equal(StatusCode.AlreadyExists, ex.StatusCode);
|
||||
Assert.True(await call2.ResponseStream.MoveNext(cts.Token));
|
||||
Assert.Equal(SyncMessage.MsgOneofCase.Handshake, call2.ResponseStream.Current.MsgCase);
|
||||
|
||||
// The incumbent really ended: the displaced client notices and dials again.
|
||||
await WaitForAsync(() => Task.FromResult(bg.ConnectionAttempts > attemptsBefore), RunTimeout);
|
||||
|
||||
await bg.StopAsync(CancellationToken.None);
|
||||
}
|
||||
@@ -173,6 +181,71 @@ public sealed class GrpcAdapterTests : IAsyncLifetime
|
||||
Assert.True(sw.Elapsed < TimeSpan.FromSeconds(3), $"teardown took {sw.Elapsed}");
|
||||
}
|
||||
|
||||
// ---- inbox backpressure ------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Inbox_SlowReader_StopsDrainingTheStream_AndLosesNothing()
|
||||
{
|
||||
// The pump must STOP pulling from the gRPC stream once the inbox is full — that stall is
|
||||
// what closes the peer's flow-control window and bounds this process's memory at
|
||||
// InboxCapacity x MaxBatchBytes instead of "whatever the peer decides to send" (a full
|
||||
// snapshot of a large dataset, on both nodes at once, when a pair baselines late).
|
||||
const int capacity = Replication.Internal.GrpcSyncDuplex.InboxCapacity;
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15));
|
||||
var reader = new CountingReader();
|
||||
var (duplex, readerTask) = Replication.Internal.GrpcSyncDuplex.Create(
|
||||
reader, new HangingWriter(), cts.Token);
|
||||
|
||||
// Nothing reads the inbox: the pump fills it and blocks.
|
||||
await WaitForAsync(() => Task.FromResult(reader.Produced >= capacity), TimeSpan.FromSeconds(5));
|
||||
await Task.Delay(200, cts.Token);
|
||||
// capacity in the channel + at most one in the pump's hand, blocked on WriteAsync.
|
||||
Assert.True(reader.Produced <= capacity + 1,
|
||||
$"the reader pump kept draining an already-full inbox: {reader.Produced} messages read for a {capacity}-message bound");
|
||||
|
||||
// Draining releases it, and every message arrives exactly once in stream order — TryWrite
|
||||
// on a full bounded channel would have dropped them silently instead.
|
||||
for (long expected = 1; expected <= capacity + 1; expected++)
|
||||
{
|
||||
var msg = await ReadInboxAsync(duplex, cts.Token);
|
||||
Assert.Equal(expected, msg.DeltaAck.AppliedThruSeq);
|
||||
}
|
||||
await WaitForAsync(() => Task.FromResult(reader.Produced > capacity + 1), TimeSpan.FromSeconds(5));
|
||||
|
||||
cts.Cancel();
|
||||
try { await readerTask.WaitAsync(TimeSpan.FromSeconds(5)); }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
|
||||
private static async Task<SyncMessage> ReadInboxAsync(
|
||||
Replication.Internal.SyncDuplex duplex, CancellationToken ct)
|
||||
{
|
||||
await duplex.Inbox.WaitToReadAsync(ct);
|
||||
duplex.Inbox.TryRead(out var message);
|
||||
return message!;
|
||||
}
|
||||
|
||||
// Produces messages as fast as the inbox will take them, numbering each one so the consumer can
|
||||
// prove nothing was dropped or reordered.
|
||||
private sealed class CountingReader : IAsyncStreamReader<SyncMessage>
|
||||
{
|
||||
private long _produced;
|
||||
|
||||
public long Produced => Interlocked.Read(ref _produced);
|
||||
|
||||
public SyncMessage Current { get; private set; } = new();
|
||||
|
||||
public Task<bool> MoveNext(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
Current = new SyncMessage
|
||||
{
|
||||
DeltaAck = new DeltaAck { AppliedThruSeq = Interlocked.Increment(ref _produced) },
|
||||
};
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- validator -------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
@@ -197,9 +270,13 @@ public sealed class GrpcAdapterTests : IAsyncLifetime
|
||||
TombstoneRetention = TimeSpan.Zero,
|
||||
ReconnectBackoffMax = TimeSpan.Zero,
|
||||
MaxHlcDriftAhead = TimeSpan.Zero,
|
||||
KeepAlivePingDelay = TimeSpan.Zero,
|
||||
KeepAlivePingTimeout = TimeSpan.Zero,
|
||||
});
|
||||
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains(result.Failures, f => f.Contains("KeepAlivePingDelay", StringComparison.Ordinal));
|
||||
Assert.Contains(result.Failures, f => f.Contains("KeepAlivePingTimeout", StringComparison.Ordinal));
|
||||
Assert.Contains(result.Failures, f => f.Contains("FlushInterval", StringComparison.Ordinal));
|
||||
Assert.Contains(result.Failures, f => f.Contains("MaxBatchSize", StringComparison.Ordinal));
|
||||
Assert.Contains(result.Failures, f => f.Contains("MaxOplogRows", StringComparison.Ordinal));
|
||||
|
||||
@@ -51,12 +51,118 @@ public sealed class LocalDbSchemaTests : IDisposable
|
||||
foreach (var expected in new[]
|
||||
{
|
||||
"__localdb_meta", "__localdb_oplog", "__localdb_peer_state",
|
||||
"__localdb_row_version", "__localdb_applying", "__localdb_dead_letter"
|
||||
"__localdb_row_version", "__localdb_applying", "__localdb_dead_letter",
|
||||
"__localdb_snapshot_state",
|
||||
})
|
||||
Assert.Contains(expected, tables);
|
||||
|
||||
// __localdb_oplog_hlc is deliberately NOT created: schema v2 removed it. It indexed
|
||||
// (table_name, pk_json, hlc), which no query has ever used — every oplog access path is by
|
||||
// seq or a full hlc-predicate delete — while costing a b-tree insert inside every captured
|
||||
// consumer write.
|
||||
Assert.Equal(0, ScalarLong(conn,
|
||||
"SELECT count(*) FROM sqlite_master WHERE type='index' AND name='__localdb_oplog_hlc'"));
|
||||
Assert.Equal(LocalDbSchema.CurrentSchemaVersion,
|
||||
ScalarLong(conn, "SELECT schema_version FROM __localdb_meta WHERE id=1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureCreated_UpgradesLegacyFile_DropsOplogHlcIndex()
|
||||
{
|
||||
using var conn = Open();
|
||||
|
||||
// A file exactly as schema v1 left it: the tables, the index, and a v1 stamp.
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
Exec(conn, "CREATE INDEX __localdb_oplog_hlc ON __localdb_oplog(table_name, pk_json, hlc);" +
|
||||
"UPDATE __localdb_meta SET schema_version = 1 WHERE id = 1;");
|
||||
Assert.Equal(1, ScalarLong(conn,
|
||||
"SELECT count(*) FROM sqlite_master WHERE type='index' AND name='__localdb_oplog_hlc'"));
|
||||
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
|
||||
Assert.Equal(0, ScalarLong(conn,
|
||||
"SELECT count(*) FROM sqlite_master WHERE type='index' AND name='__localdb_oplog_hlc'"));
|
||||
Assert.Equal(LocalDbSchema.CurrentSchemaVersion,
|
||||
ScalarLong(conn, "SELECT schema_version FROM __localdb_meta WHERE id=1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureCreated_UpgradesV2File_AddsPerTableSnapshotState()
|
||||
{
|
||||
using var conn = Open();
|
||||
|
||||
// A file exactly as schema v2 left it: everything except __localdb_snapshot_state, stamped 2.
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
Exec(conn, "DROP TABLE __localdb_snapshot_state;" +
|
||||
"UPDATE __localdb_meta SET schema_version = 2 WHERE id = 1;");
|
||||
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
|
||||
Assert.Equal(1, ScalarLong(conn,
|
||||
"SELECT count(*) FROM sqlite_master WHERE type='table' AND name='__localdb_snapshot_state'"));
|
||||
// Empty is the right v3 state for a file that never baselined anything: no table owes a
|
||||
// snapshot, and the database-wide flag it was carrying keeps whatever value it had.
|
||||
Assert.Equal(0, ScalarLong(conn, "SELECT COUNT(*) FROM __localdb_snapshot_state"));
|
||||
Assert.Equal(LocalDbSchema.CurrentSchemaVersion,
|
||||
ScalarLong(conn, "SELECT schema_version FROM __localdb_meta WHERE id=1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureCreated_Upgrade_PreservesTheDatabaseWideSnapshotFlag()
|
||||
{
|
||||
// A v2 file mid-resync (cap prune flagged a full snapshot, the peer was down) must still owe
|
||||
// that snapshot after the upgrade. v3 narrows where NEW debt is recorded; it does not
|
||||
// reinterpret debt already on disk.
|
||||
using var conn = Open();
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
Exec(conn, "DROP TABLE __localdb_snapshot_state;" +
|
||||
"UPDATE __localdb_meta SET schema_version = 2 WHERE id = 1;" +
|
||||
"UPDATE __localdb_peer_state SET needs_snapshot = 1 WHERE id = 1;");
|
||||
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
|
||||
Assert.Equal(1, ScalarLong(conn, "SELECT needs_snapshot FROM __localdb_peer_state WHERE id=1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureCreated_Upgrade_PreservesNodeIdentityAndOplogRows()
|
||||
{
|
||||
using var conn = Open();
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
var nodeId = ScalarString(conn, "SELECT node_id FROM __localdb_meta WHERE id=1");
|
||||
Exec(conn,
|
||||
"CREATE INDEX __localdb_oplog_hlc ON __localdb_oplog(table_name, pk_json, hlc);" +
|
||||
"UPDATE __localdb_meta SET schema_version = 1 WHERE id = 1;" +
|
||||
"INSERT INTO __localdb_oplog (table_name, pk_json, row_json, hlc, node_id, is_tombstone) " +
|
||||
"VALUES ('orders', '{\"id\":1}', '{\"id\":1}', 42, 'n', 0);" +
|
||||
"UPDATE __localdb_peer_state SET last_acked_seq = 7 WHERE id = 1;");
|
||||
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
|
||||
// The upgrade is DDL only — identity, replication watermarks and captured data survive it.
|
||||
Assert.Equal(nodeId, ScalarString(conn, "SELECT node_id FROM __localdb_meta WHERE id=1"));
|
||||
Assert.Equal(1, ScalarLong(conn, "SELECT COUNT(*) FROM __localdb_oplog"));
|
||||
Assert.Equal(42, ScalarLong(conn, "SELECT hlc FROM __localdb_oplog"));
|
||||
Assert.Equal(7, ScalarLong(conn, "SELECT last_acked_seq FROM __localdb_peer_state WHERE id=1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureCreated_AlreadyCurrent_LeavesVersionAlone()
|
||||
{
|
||||
using var conn = Open();
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
LocalDbSchema.EnsureCreated(conn);
|
||||
|
||||
Assert.Equal(LocalDbSchema.CurrentSchemaVersion,
|
||||
ScalarLong(conn, "SELECT schema_version FROM __localdb_meta WHERE id=1"));
|
||||
}
|
||||
|
||||
private static void Exec(SqliteConnection conn, string sql)
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = sql;
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -404,6 +404,133 @@ public sealed class LwwApplierTests : IDisposable
|
||||
Assert.Equal(("NEW", (long?)7), await ReadOrder(db, 2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Apply_PreparesEachStatementOnce_NotOncePerRow()
|
||||
{
|
||||
// The regression this guards: the old shape went through the generic transaction façade,
|
||||
// which builds a fresh SqliteCommand — and therefore re-parses the SQL — per call. Three
|
||||
// statements per applied row means a 200-row batch parsed 600 statements to run 6 distinct
|
||||
// ones. Statement count must track distinct SQL, not rows.
|
||||
using var db = await NewOrdersDb();
|
||||
await db.ExecuteAsync("CREATE TABLE widgets (id INTEGER PRIMARY KEY, name TEXT)");
|
||||
db.RegisterReplicated("widgets");
|
||||
|
||||
var entries = new List<OplogEntryRecord>();
|
||||
for (var i = 1; i <= 200; i++)
|
||||
entries.Add(Entry(i, $"{{\"id\":{i}}}", $"{{\"id\":{i},\"sku\":\"S\",\"qty\":{i}}}", 1_000_000 + i, "remote"));
|
||||
// A second table and a tombstone, so every statement kind in the set is exercised.
|
||||
entries.Add(new OplogEntryRecord(
|
||||
201, "widgets", "{\"id\":1}", "{\"id\":1,\"name\":\"W\"}", 1_000_300, "remote", false));
|
||||
entries.Add(Entry(202, "{\"id\":1}", null, 1_000_400, "remote", tombstone: true));
|
||||
|
||||
var applier = new LwwApplier(db);
|
||||
var result = await applier.ApplyBatchAsync(entries, default);
|
||||
|
||||
Assert.Equal(202, result.Applied);
|
||||
// Six table-independent statements (applying guard, version lookup, row-version write,
|
||||
// peer state, plus the per-entry savepoint's SAVEPOINT and RELEASE), an upsert per table
|
||||
// touched, and a delete for the one table tombstoned. The savepoint's ROLLBACK TO and the
|
||||
// dead-letter insert are absent because this batch has no poison entry — statements are
|
||||
// built on first use, so the count tracks the statement KINDS the batch actually ran.
|
||||
Assert.Equal(9, applier.LastBatchStatementCount);
|
||||
|
||||
// ...and the batch is still correct, statement reuse notwithstanding.
|
||||
Assert.Null(await ReadOrder(db, 1));
|
||||
Assert.Equal(("S", (long?)200), await ReadOrder(db, 200));
|
||||
var widget = await db.QueryAsync("SELECT name FROM widgets WHERE id = 1", x => x.GetString(0));
|
||||
Assert.Equal("W", widget[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Apply_ReusedStatements_SurviveAPoisonRowMidBatch()
|
||||
{
|
||||
// A statement that faults is reset and reused for the following rows; if it were not, one
|
||||
// poison row would silently corrupt every row after it in the batch.
|
||||
using var db = await NewOrdersDb();
|
||||
|
||||
var applier = new LwwApplier(db);
|
||||
var result = await applier.ApplyBatchAsync(
|
||||
[
|
||||
Entry(1, "{\"id\":1}", "{\"id\":1,\"sku\":\"BEFORE\",\"qty\":1}", 1_000_000, "remote"),
|
||||
Entry(2, "{\"id\":2}", "NOT VALID JSON", 1_000_001, "remote"),
|
||||
Entry(3, "{\"id\":3}", "{\"id\":3,\"sku\":\"AFTER\",\"qty\":3}", 1_000_002, "remote"),
|
||||
Entry(4, "{\"id\":4}", "ALSO NOT JSON", 1_000_003, "remote"),
|
||||
Entry(5, "{\"id\":5}", "{\"id\":5,\"sku\":\"LAST\",\"qty\":5}", 1_000_004, "remote"),
|
||||
], default);
|
||||
|
||||
Assert.Equal(3, result.Applied);
|
||||
Assert.Equal(2, result.DeadLettered);
|
||||
Assert.Equal(("BEFORE", (long?)1), await ReadOrder(db, 1));
|
||||
Assert.Equal(("AFTER", (long?)3), await ReadOrder(db, 3));
|
||||
Assert.Equal(("LAST", (long?)5), await ReadOrder(db, 5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Apply_LedgerWriteFails_LeavesNeitherTheDeleteNorTheLedgerEntry()
|
||||
{
|
||||
// The hazard (P4-7): applying a tombstone is TWO statements — delete the row, then write the
|
||||
// ledger's tombstone version — and the poison catch commits whatever the first one already
|
||||
// did. A failure landing between them therefore committed a DELETED ROW carrying a STALE LIVE
|
||||
// ledger entry, which is exactly the shape SnapshotStreamer warns about and skips: the key
|
||||
// then never travels again and the pair is divergent for that row, permanently and silently.
|
||||
//
|
||||
// The failure is injected with a trigger that aborts the ledger write (in production the
|
||||
// realistic causes are disk-full / IO error / corruption — none of them reproducible here),
|
||||
// and RAISE(ABORT) aborts only the statement, so the applier's narrow SqliteException catch
|
||||
// treats it as per-entry poison exactly as it would a real one.
|
||||
using var db = await NewOrdersDb();
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1, 'LOCAL', 5)");
|
||||
var localHlc = await RowVersionHlc(db, Pk1);
|
||||
|
||||
await db.ExecuteAsync(
|
||||
"CREATE TRIGGER poison_ledger BEFORE INSERT ON __localdb_row_version " +
|
||||
"WHEN NEW.is_tombstone = 1 BEGIN SELECT RAISE(ABORT, 'injected ledger failure'); END");
|
||||
|
||||
var result = await new LwwApplier(db).ApplyBatchAsync(
|
||||
[Entry(90, Pk1, null, localHlc + 1000, "remote-node", tombstone: true)], default);
|
||||
|
||||
// The entry is poison, so it is dead-lettered and the batch continues — unchanged semantics.
|
||||
Assert.Equal(1, result.DeadLettered);
|
||||
Assert.Equal(0, result.Applied);
|
||||
|
||||
// BOTH or NEITHER. Neither, here: the row must still be present, because its ledger entry
|
||||
// still says "live at the local version".
|
||||
Assert.Equal(("LOCAL", (long?)5), await ReadOrder(db, 1));
|
||||
var rv = await db.QueryAsync(
|
||||
"SELECT hlc, is_tombstone FROM __localdb_row_version WHERE table_name = 'orders' AND pk_json = @pk",
|
||||
x => (Hlc: x.GetInt64(0), Tomb: x.GetInt64(1)), new { pk = Pk1 });
|
||||
Assert.Single(rv);
|
||||
Assert.Equal(localHlc, rv[0].Hlc);
|
||||
Assert.Equal(0, rv[0].Tomb);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Apply_LedgerWriteFails_MidBatch_LeavesTheOtherEntriesApplied()
|
||||
{
|
||||
// The atomicity must be per-ENTRY, not per-batch: rolling back to the savepoint may not
|
||||
// discard the rows applied before the poison one, nor block the ones after it.
|
||||
using var db = await NewOrdersDb();
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (2, 'DOOMED', 2)");
|
||||
var doomedHlc = await RowVersionHlc(db, "{\"id\":2}");
|
||||
|
||||
await db.ExecuteAsync(
|
||||
"CREATE TRIGGER poison_ledger BEFORE INSERT ON __localdb_row_version " +
|
||||
"WHEN NEW.is_tombstone = 1 BEGIN SELECT RAISE(ABORT, 'injected ledger failure'); END");
|
||||
|
||||
var result = await new LwwApplier(db).ApplyBatchAsync(
|
||||
[
|
||||
Entry(91, Pk1, "{\"id\":1,\"sku\":\"BEFORE\",\"qty\":1}", doomedHlc + 1, "remote-node"),
|
||||
Entry(92, "{\"id\":2}", null, doomedHlc + 1000, "remote-node", tombstone: true),
|
||||
Entry(93, "{\"id\":3}", "{\"id\":3,\"sku\":\"AFTER\",\"qty\":3}", doomedHlc + 3, "remote-node"),
|
||||
], default);
|
||||
|
||||
Assert.Equal(2, result.Applied);
|
||||
Assert.Equal(1, result.DeadLettered);
|
||||
Assert.Equal(("BEFORE", (long?)1), await ReadOrder(db, 1));
|
||||
Assert.Equal(("DOOMED", (long?)2), await ReadOrder(db, 2));
|
||||
Assert.Equal(("AFTER", (long?)3), await ReadOrder(db, 3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Apply_ReplaceExistingRow_DoesNotFireCaptureTriggers()
|
||||
{
|
||||
|
||||
@@ -212,6 +212,137 @@ public sealed class OplogStoreTests : IDisposable
|
||||
Assert.Equal("{\"id\":2}", remaining[0].PkJson);
|
||||
}
|
||||
|
||||
// ---- O(1) backlog depth --------------------------------------------------------------
|
||||
|
||||
// The definition the O(1) formula has to reproduce. Every scenario below is asserted against
|
||||
// this, not against a hand-computed number, so the two can never drift apart.
|
||||
private static async Task<long> CountingDepth(SqliteLocalDb db) =>
|
||||
(await db.QueryAsync(
|
||||
"SELECT COUNT(*) FROM __localdb_oplog WHERE seq > (SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1)",
|
||||
r => r.GetInt64(0)))[0];
|
||||
|
||||
private static async Task AssertDepthMatchesCount(SqliteLocalDb db, OplogStore store)
|
||||
{
|
||||
var expected = await CountingDepth(db);
|
||||
Assert.Equal(expected, await store.GetOplogDepthAsync(default));
|
||||
Assert.Equal(expected, store.GetOplogDepthSync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OplogDepth_Empty_IsZero()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
var store = new OplogStore(db, new ReplicationOptions());
|
||||
|
||||
await AssertDepthMatchesCount(db, store);
|
||||
Assert.Equal(0, await store.GetOplogDepthAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OplogDepth_AllAcked_EmptiesToZero_NotToTheSeqRange()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
await WriteRows(db, 5);
|
||||
var store = new OplogStore(db, new ReplicationOptions());
|
||||
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync(default), default);
|
||||
|
||||
await AssertDepthMatchesCount(db, store);
|
||||
Assert.Equal(0, await store.GetOplogDepthAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OplogDepth_AfterCapPrune_ReportsRetainedRows_NotTheWholeSeqRange()
|
||||
{
|
||||
// The reason the formula clamps its floor to MIN(seq) - 1. A cap prune leaves the retained
|
||||
// rows far above an ack watermark that never moved, so a naive MAX(seq) - last_acked would
|
||||
// report the entire seq range forever and re-flag the cap on every maintenance tick.
|
||||
using var db = await NewOrdersDb();
|
||||
await WriteRows(db, 40);
|
||||
var store = new OplogStore(db, new ReplicationOptions { MaxOplogRows = 5 });
|
||||
|
||||
Assert.True(await store.EnforceCapsAsync(default));
|
||||
|
||||
await AssertDepthMatchesCount(db, store);
|
||||
Assert.Equal(5, await store.GetOplogDepthAsync(default));
|
||||
// And the next tick is clean: depth is not over cap any more.
|
||||
Assert.False(await store.EnforceCapsAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OplogDepth_AfterAgePrune_MatchesCount()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
await WriteRows(db, 6);
|
||||
// Backdate the oldest three past the age cap; HLCs rise with seq, so this is a prefix.
|
||||
var expired = DateTimeOffset.UtcNow.AddDays(-30).ToUnixTimeMilliseconds() << 16;
|
||||
await db.ExecuteAsync(
|
||||
"UPDATE __localdb_oplog SET hlc = @h WHERE seq IN (SELECT seq FROM __localdb_oplog ORDER BY seq LIMIT 3)",
|
||||
new { h = expired });
|
||||
var store = new OplogStore(db, new ReplicationOptions());
|
||||
|
||||
Assert.True(await store.EnforceCapsAsync(default));
|
||||
|
||||
await AssertDepthMatchesCount(db, store);
|
||||
Assert.Equal(3, await store.GetOplogDepthAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OplogDepth_PartialAck_MatchesCount()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
await WriteRows(db, 9);
|
||||
var store = new OplogStore(db, new ReplicationOptions());
|
||||
var batch = await store.ReadBatchAboveAsync(0, 100, default);
|
||||
|
||||
await store.RecordPeerAckAsync(batch[3].Seq, default);
|
||||
|
||||
await AssertDepthMatchesCount(db, store);
|
||||
Assert.Equal(5, await store.GetOplogDepthAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OplogDepth_AckPruneThenMoreWrites_MatchesCount()
|
||||
{
|
||||
using var db = await NewOrdersDb();
|
||||
await WriteRows(db, 4);
|
||||
var store = new OplogStore(db, new ReplicationOptions());
|
||||
await store.RecordPeerAckAsync(await store.GetMaxSeqAsync(default), default);
|
||||
|
||||
// The steady state of a converged pair: oplog pruned empty, watermark high, new writes above it.
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (91,'S',1),(92,'S',2)");
|
||||
|
||||
await AssertDepthMatchesCount(db, store);
|
||||
Assert.Equal(2, await store.GetOplogDepthAsync(default));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OplogDepth_LargeBacklog_IsConstantTime()
|
||||
{
|
||||
// 200k rows: a COUNT(*) over the backlog runs ~200k VM steps per call (measured: 100,020 at
|
||||
// 50k rows), the seek-based formula runs 58 regardless of size. The loop below costs tens of
|
||||
// milliseconds at seek shape and seconds at scan shape, so the budget separates them by a
|
||||
// wide margin in both directions.
|
||||
using var db = await NewOrdersDb();
|
||||
await db.ExecuteAsync(
|
||||
"INSERT INTO __localdb_oplog (table_name, pk_json, row_json, hlc, node_id, is_tombstone) " +
|
||||
"WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 200000) " +
|
||||
"SELECT 'orders', '{\"id\":' || x || '}', '{\"id\":' || x || '}', x << 16, 'n', 0 FROM c");
|
||||
var store = new OplogStore(db, new ReplicationOptions());
|
||||
|
||||
Assert.Equal(200_000, await store.GetOplogDepthAsync(default));
|
||||
|
||||
using var conn = db.CreateConnection();
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = OplogStore.DepthSql;
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
for (var i = 0; i < 2_000; i++)
|
||||
Assert.Equal(200_000L, (long)cmd.ExecuteScalar()!);
|
||||
sw.Stop();
|
||||
|
||||
Assert.True(sw.ElapsedMilliseconds < 1_500,
|
||||
$"2000 depth reads over a 200k backlog took {sw.ElapsedMilliseconds} ms — that is scan-shaped, not seek-shaped.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PeerState_RoundTrips()
|
||||
{
|
||||
|
||||
@@ -70,6 +70,97 @@ public sealed class RegistrationTests : IDisposable
|
||||
Assert.Contains("BLOB", ex.Message);
|
||||
}
|
||||
|
||||
// ---- Cascading-delete foreign-key parents (P4-7) --------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Register_CascadeDeleteFkParent_Throws()
|
||||
{
|
||||
// The hazard: applying a remote row uses INSERT OR REPLACE, whose implicit DELETE fires
|
||||
// ON DELETE CASCADE while the capture guard is RAISED — so the cascaded child rows are
|
||||
// deleted locally and never captured, and the peer keeps them forever.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync(
|
||||
"CREATE TABLE order_lines (id INTEGER PRIMARY KEY, order_id INTEGER " +
|
||||
"REFERENCES orders(id) ON DELETE CASCADE)");
|
||||
|
||||
var ex = Assert.Throws<LocalDbRegistrationException>(() => db.RegisterReplicated("orders"));
|
||||
Assert.Contains("orders", ex.Message);
|
||||
Assert.Contains("order_lines", ex.Message);
|
||||
Assert.Contains("ON DELETE CASCADE", ex.Message);
|
||||
|
||||
// Refused means refused: no triggers installed, nothing in the registry.
|
||||
Assert.Empty(await Triggers(db));
|
||||
Assert.False(db.ReplicatedTables.ContainsKey("orders"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_CascadeDeleteFkChild_IsAllowed()
|
||||
{
|
||||
// Only the PARENT is refused. The child side is safe: replacing a child row cascades
|
||||
// nothing, and the child's own deletes are captured normally.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync(
|
||||
"CREATE TABLE order_lines (id INTEGER PRIMARY KEY, order_id INTEGER " +
|
||||
"REFERENCES orders(id) ON DELETE CASCADE)");
|
||||
|
||||
db.RegisterReplicated("order_lines");
|
||||
|
||||
Assert.True(db.ReplicatedTables.ContainsKey("order_lines"));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("ON DELETE SET NULL", "ON DELETE SET NULL")]
|
||||
[InlineData("ON DELETE SET DEFAULT", "ON DELETE SET DEFAULT")]
|
||||
public async Task Register_OtherRowMutatingDeleteActions_AlsoThrow(string clause, string expected)
|
||||
{
|
||||
// SET NULL / SET DEFAULT mutate child rows on the same implicit DELETE and are captured
|
||||
// just as little — same divergence, so the same refusal.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync(
|
||||
"CREATE TABLE order_lines (id INTEGER PRIMARY KEY, order_id INTEGER DEFAULT 0 " +
|
||||
$"REFERENCES orders(id) {clause})");
|
||||
|
||||
var ex = Assert.Throws<LocalDbRegistrationException>(() => db.RegisterReplicated("orders"));
|
||||
Assert.Contains(expected, ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_NonCascadingFkParent_IsAllowed()
|
||||
{
|
||||
// The default (NO ACTION / RESTRICT) mutates no child row: the implicit DELETE simply
|
||||
// fails the statement, which the applier already surfaces as a dead-lettered entry. Noisy,
|
||||
// not silent — so there is nothing to refuse.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync(
|
||||
"CREATE TABLE order_lines (id INTEGER PRIMARY KEY, order_id INTEGER REFERENCES orders(id))");
|
||||
await db.ExecuteAsync("CREATE TABLE restricted (id INTEGER PRIMARY KEY, " +
|
||||
"order_id INTEGER REFERENCES orders(id) ON DELETE RESTRICT)");
|
||||
|
||||
db.RegisterReplicated("orders");
|
||||
|
||||
Assert.Equal(3, (await Triggers(db)).Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_UnrelatedTablesWithCascades_DoNotBlockRegistration()
|
||||
{
|
||||
// The scan must key on the table being registered, not on "the database contains a cascade".
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync("CREATE TABLE widgets (id INTEGER PRIMARY KEY, name TEXT)");
|
||||
await db.ExecuteAsync(
|
||||
"CREATE TABLE widget_parts (id INTEGER PRIMARY KEY, widget_id INTEGER " +
|
||||
"REFERENCES widgets(id) ON DELETE CASCADE)");
|
||||
|
||||
db.RegisterReplicated("orders");
|
||||
|
||||
Assert.True(db.ReplicatedTables.ContainsKey("orders"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_IsIdempotent_RegeneratesTriggers()
|
||||
{
|
||||
@@ -128,4 +219,409 @@ public sealed class RegistrationTests : IDisposable
|
||||
Assert.Equal(new[] { "id" }, table.PkColumns);
|
||||
Assert.Equal(new[] { "id", "sku", "qty" }, table.Columns.Select(c => c.Name).ToArray());
|
||||
}
|
||||
|
||||
// ---- Deregistration ------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_DropsTriggers_PrunesOplogAndLedger_LeavesDataAlone()
|
||||
{
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
db.RegisterReplicated("orders");
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1),(2,'B',2)");
|
||||
|
||||
var removed = db.DeregisterReplicated("orders");
|
||||
|
||||
Assert.True(removed);
|
||||
Assert.Empty(await Triggers(db));
|
||||
Assert.Equal(0, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name='orders'"));
|
||||
Assert.Equal(0, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_row_version WHERE table_name='orders'"));
|
||||
Assert.False(db.ReplicatedTables.ContainsKey("orders"));
|
||||
|
||||
// The consumer's own data is never touched — this removes CAPTURE, not rows.
|
||||
Assert.Equal(2, await CountAsync(db, "SELECT COUNT(*) FROM orders"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_StopsCapture()
|
||||
{
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
db.RegisterReplicated("orders");
|
||||
db.DeregisterReplicated("orders");
|
||||
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
await db.ExecuteAsync("UPDATE orders SET qty = 9 WHERE id = 1");
|
||||
await db.ExecuteAsync("DELETE FROM orders WHERE id = 1");
|
||||
|
||||
Assert.Equal(0, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_oplog"));
|
||||
Assert.Equal(0, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_row_version"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_IsIdempotent_SecondCallReportsNothingToDo()
|
||||
{
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
db.RegisterReplicated("orders");
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
|
||||
Assert.True(db.DeregisterReplicated("orders"));
|
||||
Assert.False(db.DeregisterReplicated("orders"));
|
||||
Assert.False(db.DeregisterReplicated("orders"));
|
||||
Assert.Empty(await Triggers(db));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_NeverRegisteredTable_IsANoOp()
|
||||
{
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
|
||||
Assert.False(db.DeregisterReplicated("orders"));
|
||||
Assert.False(db.DeregisterReplicated("ghost"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_ClearsTriggersLeftByAnEarlierProcess()
|
||||
{
|
||||
// The primary use: a database file registered by an OLDER build, opened by one that has
|
||||
// decided this node does not replicate. Nothing in memory says the table was ever
|
||||
// registered — the evidence is entirely on disk.
|
||||
using (var first = Create())
|
||||
{
|
||||
await CreateOrders(first);
|
||||
first.RegisterReplicated("orders");
|
||||
await first.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
}
|
||||
SqliteConnection.ClearAllPools();
|
||||
|
||||
using var second = Create();
|
||||
Assert.Equal(3, (await Triggers(second)).Count);
|
||||
Assert.False(second.ReplicatedTables.ContainsKey("orders"));
|
||||
|
||||
Assert.True(second.DeregisterReplicated("orders"));
|
||||
|
||||
Assert.Empty(await Triggers(second));
|
||||
Assert.Equal(0, await CountAsync(second, "SELECT COUNT(*) FROM __localdb_oplog"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_LeavesOtherTablesCapturing()
|
||||
{
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync("CREATE TABLE widgets (id INTEGER PRIMARY KEY, name TEXT)");
|
||||
db.RegisterReplicated("orders");
|
||||
db.RegisterReplicated("widgets");
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
await db.ExecuteAsync("INSERT INTO widgets (id, name) VALUES (1,'W')");
|
||||
|
||||
db.DeregisterReplicated("orders");
|
||||
await db.ExecuteAsync("INSERT INTO widgets (id, name) VALUES (2,'W2')");
|
||||
|
||||
Assert.Equal(0, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name='orders'"));
|
||||
Assert.Equal(2, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name='widgets'"));
|
||||
Assert.True(db.ReplicatedTables.ContainsKey("widgets"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_ThenRegisterWithBaseline_RestoresTheLedger()
|
||||
{
|
||||
// Pruning the ledger is only safe because re-registration can re-baseline; this is that
|
||||
// round trip.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
db.RegisterReplicated("orders");
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1),(2,'B',2)");
|
||||
|
||||
db.DeregisterReplicated("orders");
|
||||
Assert.Equal(0, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_row_version"));
|
||||
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
Assert.Equal(2, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_row_version WHERE table_name='orders'"));
|
||||
}
|
||||
|
||||
// ---- Baselining ----------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Register_WithoutBaseline_LeavesPreexistingRowsUnledgered()
|
||||
{
|
||||
// The documented CDC contract, unchanged: opting in is what changes behaviour, not upgrading.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
|
||||
db.RegisterReplicated("orders");
|
||||
|
||||
Assert.Equal(0, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_row_version"));
|
||||
Assert.False(await NeedsSnapshot(db));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_WithBaseline_SeedsAtHlcZero_AndFlagsSnapshot()
|
||||
{
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1),(2,'B',2)");
|
||||
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
var seeded = await db.QueryAsync(
|
||||
"SELECT pk_json, hlc, node_id, is_tombstone FROM __localdb_row_version ORDER BY pk_json",
|
||||
r => (Pk: r.GetString(0), Hlc: r.GetInt64(1), Node: r.GetString(2), Tomb: r.GetInt64(3)));
|
||||
|
||||
Assert.Equal(2, seeded.Count);
|
||||
Assert.Equal(["{\"id\":1}", "{\"id\":2}"], seeded.Select(s => s.Pk));
|
||||
// HLC 0 is the floor: below every genuine stamp (physical ms << 16), so it can never beat one.
|
||||
Assert.All(seeded, s => Assert.Equal(0, s.Hlc));
|
||||
Assert.All(seeded, s => Assert.Equal(db.NodeId, s.Node));
|
||||
Assert.All(seeded, s => Assert.Equal(0, s.Tomb));
|
||||
|
||||
// Without the flag the seeding would be inert: no gap exists at the handshake, so the peer
|
||||
// would be told it needs no snapshot and the baselined rows would never travel.
|
||||
Assert.True(await NeedsSnapshot(db));
|
||||
// ...and it is flagged for THIS TABLE ONLY. Through 0.2.x this set the one database-wide
|
||||
// bit, so baselining `orders` re-streamed every other registered table with it.
|
||||
Assert.False(await NeedsFullSnapshot(db));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_WithBaseline_FlagsOnlyTheBaselinedTable()
|
||||
{
|
||||
// The per-table flag's reason to exist: opting one table into replication late must not
|
||||
// schedule a full re-stream of its unrelated neighbours.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync("CREATE TABLE widgets (id INTEGER PRIMARY KEY, name TEXT)");
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
await db.ExecuteAsync("INSERT INTO widgets (id, name) VALUES (1,'W')");
|
||||
|
||||
db.RegisterReplicated("widgets");
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
Assert.True(await NeedsSnapshot(db, "orders"));
|
||||
Assert.False(await NeedsSnapshot(db, "widgets"));
|
||||
Assert.False(await NeedsFullSnapshot(db));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_DropsThePendingSnapshotFlag()
|
||||
{
|
||||
// A flag naming a table nothing streams any more can never clear itself — the streamer skips
|
||||
// unregistered tables — so it would sit in the file and re-appear as unasked-for debt if the
|
||||
// table were registered again.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
Assert.True(await NeedsSnapshot(db));
|
||||
|
||||
db.DeregisterReplicated("orders");
|
||||
|
||||
Assert.False(await NeedsSnapshot(db));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_WithBaseline_IsIdempotent_AndDoesNotReflagSnapshot()
|
||||
{
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
// Stand in for the snapshot having been delivered and acknowledged.
|
||||
await db.ExecuteAsync("DELETE FROM __localdb_snapshot_state");
|
||||
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
Assert.Equal(1, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_row_version"));
|
||||
// Nothing new was seeded, so no resync is owed: passing true every boot is free after the first.
|
||||
Assert.False(await NeedsSnapshot(db));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_WithBaseline_NeverOverwritesACapturedVersion()
|
||||
{
|
||||
// The LWW-safety core: a row with a real, captured version must keep it. A floor that
|
||||
// clobbered genuine versions would make every baselined node look newer than its peer.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
db.RegisterReplicated("orders");
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
var captured = (await db.QueryAsync(
|
||||
"SELECT hlc FROM __localdb_row_version WHERE pk_json='{\"id\":1}'", r => r.GetInt64(0)))[0];
|
||||
Assert.True(captured > 0);
|
||||
|
||||
// A second, pre-existing row that capture never saw (inserted with triggers removed).
|
||||
db.DeregisterReplicated("orders");
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (2,'B',2)");
|
||||
await db.ExecuteAsync(
|
||||
"INSERT INTO __localdb_row_version (table_name, pk_json, hlc, node_id, is_tombstone) " +
|
||||
"VALUES ('orders', '{\"id\":1}', @h, 'genuine', 0)", new { h = captured });
|
||||
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
var versions = await db.QueryAsync(
|
||||
"SELECT pk_json, hlc, node_id FROM __localdb_row_version ORDER BY pk_json",
|
||||
r => (Pk: r.GetString(0), Hlc: r.GetInt64(1), Node: r.GetString(2)));
|
||||
Assert.Equal(2, versions.Count);
|
||||
Assert.Equal(captured, versions[0].Hlc);
|
||||
Assert.Equal("genuine", versions[0].Node);
|
||||
Assert.Equal(0, versions[1].Hlc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_WithBaseline_SeedsPkJsonIdenticalToWhatCaptureWrites()
|
||||
{
|
||||
// A seeded key that differs by so much as a space from the capture triggers' json_object
|
||||
// output would be a DIFFERENT ledger row, and the baseline would silently never merge.
|
||||
using var db = Create();
|
||||
await db.ExecuteAsync("CREATE TABLE kv (a INTEGER, b TEXT, v TEXT, PRIMARY KEY (a, b))");
|
||||
await db.ExecuteAsync("INSERT INTO kv (a, b, v) VALUES (1, 'x', 'seeded')");
|
||||
|
||||
db.RegisterReplicated("kv", baselineExistingRows: true);
|
||||
await db.ExecuteAsync("UPDATE kv SET v = 'captured' WHERE a = 1 AND b = 'x'");
|
||||
|
||||
// One ledger row, not two: the capture upsert landed on the seeded key.
|
||||
var rows = await db.QueryAsync(
|
||||
"SELECT pk_json, hlc FROM __localdb_row_version", r => (Pk: r.GetString(0), Hlc: r.GetInt64(1)));
|
||||
Assert.Single(rows);
|
||||
Assert.Equal("{\"a\":1,\"b\":\"x\"}", rows[0].Pk);
|
||||
Assert.True(rows[0].Hlc > 0, "the genuine capture must have replaced the floor");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_WithBaseline_FloorLosesToEveryGenuineRemoteWrite()
|
||||
{
|
||||
// Proved through the real LWW path rather than by reasoning about the comparison: an
|
||||
// absurdly small remote HLC of 1 still beats the baseline floor of 0.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'BASELINE',1)");
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
var result = await new Replication.Internal.LwwApplier(db).ApplyBatchAsync(
|
||||
[new Replication.Internal.OplogEntryRecord(
|
||||
1, "orders", "{\"id\":1}", "{\"id\":1,\"sku\":\"REMOTE\",\"qty\":9}", 1, "peer", false)],
|
||||
default);
|
||||
|
||||
Assert.Equal(1, result.Applied);
|
||||
var sku = await db.QueryAsync("SELECT sku FROM orders WHERE id = 1", r => r.GetString(0));
|
||||
Assert.Equal("REMOTE", sku[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_WithBaseline_TieBetweenTwoBaselinedNodes_ResolvesByNodeIdOrdinal()
|
||||
{
|
||||
// Two nodes that baselined the same key both hold HLC 0 — genuinely unordered history. The
|
||||
// node-id tie-break is what keeps that convergent instead of oscillating.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'MINE',1)");
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
var applier = new Replication.Internal.LwwApplier(db);
|
||||
var lower = await applier.ApplyBatchAsync(
|
||||
[new Replication.Internal.OplogEntryRecord(
|
||||
1, "orders", "{\"id\":1}", "{\"id\":1,\"sku\":\"LOWER\",\"qty\":9}", 0, "!" + db.NodeId, false)],
|
||||
default);
|
||||
Assert.Equal(1, lower.Discarded);
|
||||
|
||||
var higher = await applier.ApplyBatchAsync(
|
||||
[new Replication.Internal.OplogEntryRecord(
|
||||
2, "orders", "{\"id\":1}", "{\"id\":1,\"sku\":\"HIGHER\",\"qty\":9}", 0, db.NodeId + "z", false)],
|
||||
default);
|
||||
Assert.Equal(1, higher.Applied);
|
||||
|
||||
var sku = await db.QueryAsync("SELECT sku FROM orders WHERE id = 1", r => r.GetString(0));
|
||||
Assert.Equal("HIGHER", sku[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Register_WithBaseline_EmptyTable_SeedsNothing_FlagsNothing()
|
||||
{
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
Assert.Equal(0, await CountAsync(db, "SELECT COUNT(*) FROM __localdb_row_version"));
|
||||
Assert.False(await NeedsSnapshot(db));
|
||||
}
|
||||
|
||||
// ---- Deregistration and the HLC crash anchor ------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_FlushesHlcAnchor_SoAnUngracefulReopenCannotRegress()
|
||||
{
|
||||
// The clock recovers as MAX(hlc_high_water, oplog HLCs, ledger HLCs, last_seen_hlc), and
|
||||
// deregistering the only registered table empties both ledgers. Without the in-transaction
|
||||
// flush, an ungraceful exit right afterwards leaves hlc_high_water at 0 and the reopened
|
||||
// clock restarts from wall-clock time — below stamps the peer already holds.
|
||||
var crashed = Create();
|
||||
try
|
||||
{
|
||||
await CreateOrders(crashed);
|
||||
crashed.RegisterReplicated("orders");
|
||||
await crashed.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'A',1)");
|
||||
|
||||
// A peer HLC an hour ahead, merged as the applier would merge it: this is what makes the
|
||||
// clock outrun wall-clock time, and therefore what a wall-clock restart would regress.
|
||||
var peerHlc = DateTimeOffset.UtcNow.AddHours(1).ToUnixTimeMilliseconds() << 16;
|
||||
crashed.Clock.Observe(peerHlc);
|
||||
var preDereg = crashed.Clock.Current;
|
||||
Assert.True(preDereg >= peerHlc);
|
||||
|
||||
crashed.DeregisterReplicated("orders");
|
||||
|
||||
Assert.Equal(preDereg, await CountAsync(crashed, "SELECT hlc_high_water FROM __localdb_meta WHERE id = 1"));
|
||||
|
||||
// The evidence the clock would otherwise have recovered from is gone.
|
||||
Assert.Equal(0, await CountAsync(crashed, "SELECT COUNT(*) FROM __localdb_oplog"));
|
||||
Assert.Equal(0, await CountAsync(crashed, "SELECT COUNT(*) FROM __localdb_row_version"));
|
||||
Assert.Equal(0, await CountAsync(crashed, "SELECT last_seen_hlc FROM __localdb_peer_state WHERE id = 1"));
|
||||
|
||||
// Ungraceful reopen: a second instance over the same file while the first was NEVER
|
||||
// disposed, so nothing but the deregistration itself can have written the anchor.
|
||||
using var reopened = Create();
|
||||
Assert.True(reopened.Clock.Current >= preDereg,
|
||||
$"recovered clock {reopened.Clock.Current} regressed below the pre-deregistration stamp {preDereg}");
|
||||
Assert.True(reopened.Clock.Next() > preDereg);
|
||||
}
|
||||
finally
|
||||
{
|
||||
crashed.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deregister_HlcFlush_DoesNotCountAsWorkDone()
|
||||
{
|
||||
// The flush is an UPDATE on every call, including the ones that clean up nothing; it must
|
||||
// not turn the "was there anything to do?" answer into a permanent true.
|
||||
using var db = Create();
|
||||
await CreateOrders(db);
|
||||
|
||||
Assert.False(db.DeregisterReplicated("orders"));
|
||||
Assert.False(db.DeregisterReplicated("ghost"));
|
||||
}
|
||||
|
||||
private static async Task<long> CountAsync(SqliteLocalDb db, string sql) =>
|
||||
(await db.QueryAsync(sql, r => r.GetInt64(0)))[0];
|
||||
|
||||
/// <summary>Whether <paramref name="table"/> individually owes the peer a snapshot (schema v3).</summary>
|
||||
private static async Task<bool> NeedsSnapshot(SqliteLocalDb db, string table = "orders") =>
|
||||
(await db.QueryAsync(
|
||||
"SELECT COUNT(*) FROM __localdb_snapshot_state WHERE table_name = @t AND needs_snapshot <> 0",
|
||||
r => r.GetInt64(0), new { t = table }))[0] != 0;
|
||||
|
||||
/// <summary>
|
||||
/// The DATABASE-WIDE flag — "every registered table". Baselining must never set it: that is the
|
||||
/// whole point of the per-table flags, and setting it would restore the 0.2.x behaviour silently.
|
||||
/// </summary>
|
||||
private static async Task<bool> NeedsFullSnapshot(SqliteLocalDb db) =>
|
||||
(await db.QueryAsync(
|
||||
"SELECT needs_snapshot FROM __localdb_peer_state WHERE id = 1", r => r.GetInt64(0)))[0] != 0;
|
||||
}
|
||||
|
||||
@@ -67,10 +67,15 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
return new Side(db, store, applier, session);
|
||||
}
|
||||
|
||||
private static (SyncDuplex A, SyncDuplex B) DuplexPair()
|
||||
// capacity models a transport that applies backpressure (which the gRPC duplex now is, via its
|
||||
// bounded inbox); null is the historical unbounded pipe.
|
||||
private static (SyncDuplex A, SyncDuplex B) DuplexPair(int? capacity = null)
|
||||
{
|
||||
var aToB = Channel.CreateUnbounded<SyncMessage>();
|
||||
var bToA = Channel.CreateUnbounded<SyncMessage>();
|
||||
Channel<SyncMessage> New() => capacity is int c
|
||||
? Channel.CreateBounded<SyncMessage>(c)
|
||||
: Channel.CreateUnbounded<SyncMessage>();
|
||||
var aToB = New();
|
||||
var bToA = New();
|
||||
var a = new SyncDuplex { Send = (m, ct) => aToB.Writer.WriteAsync(m, ct).AsTask(), Inbox = bToA.Reader };
|
||||
var b = new SyncDuplex { Send = (m, ct) => bToA.Writer.WriteAsync(m, ct).AsTask(), Inbox = aToB.Reader };
|
||||
return (a, b);
|
||||
@@ -84,9 +89,19 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
return (duplex, toSession.Writer, fromSession.Reader);
|
||||
}
|
||||
|
||||
private static Handshake MatchingHandshake(SqliteLocalDb sessionDb, string peerNodeId, long lastAppliedRemoteSeq = 0)
|
||||
// supportsPartialSnapshot defaults to FALSE, which is exactly what a 0.1.x/0.2.x peer puts on
|
||||
// the wire (the field does not exist in its proto, so proto3 decodes the default). Scripted
|
||||
// peers are therefore old peers unless a test says otherwise.
|
||||
private static Handshake MatchingHandshake(
|
||||
SqliteLocalDb sessionDb, string peerNodeId, long lastAppliedRemoteSeq = 0, bool supportsPartialSnapshot = false)
|
||||
{
|
||||
var h = new Handshake { NodeId = peerNodeId, LibSchemaVersion = 1, LastAppliedRemoteSeq = lastAppliedRemoteSeq };
|
||||
var h = new Handshake
|
||||
{
|
||||
NodeId = peerNodeId,
|
||||
LibSchemaVersion = 1,
|
||||
LastAppliedRemoteSeq = lastAppliedRemoteSeq,
|
||||
SupportsPartialSnapshot = supportsPartialSnapshot,
|
||||
};
|
||||
foreach (var t in sessionDb.ReplicatedTables.Values.OrderBy(t => t.Name, StringComparer.Ordinal))
|
||||
h.Tables.Add(new TableDigest { TableName = t.Name, Digest = t.Digest });
|
||||
return h;
|
||||
@@ -109,14 +124,34 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
|
||||
// Completes the two-phase handshake against a real session: the peer never requests a snapshot.
|
||||
private static async Task DriveHandshakeAsync(
|
||||
SqliteLocalDb sessionDb, ChannelWriter<SyncMessage> toSession, ChannelReader<SyncMessage> fromSession, CancellationToken ct)
|
||||
SqliteLocalDb sessionDb, ChannelWriter<SyncMessage> toSession, ChannelReader<SyncMessage> fromSession,
|
||||
CancellationToken ct, bool supportsPartialSnapshot = false, long lastAppliedRemoteSeq = 0)
|
||||
{
|
||||
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.Handshake, ct);
|
||||
await toSession.WriteAsync(new SyncMessage { Handshake = MatchingHandshake(sessionDb, "peer") }, ct);
|
||||
await toSession.WriteAsync(
|
||||
new SyncMessage
|
||||
{
|
||||
Handshake = MatchingHandshake(
|
||||
sessionDb, "peer", lastAppliedRemoteSeq, supportsPartialSnapshot),
|
||||
}, ct);
|
||||
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.HandshakeAck, ct);
|
||||
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, ct);
|
||||
}
|
||||
|
||||
// Reads outbound messages until (and including) the first of the given kind.
|
||||
private static async Task<List<SyncMessage>> DrainThroughAsync(
|
||||
ChannelReader<SyncMessage> reader, SyncMessage.MsgOneofCase last, CancellationToken ct)
|
||||
{
|
||||
var all = new List<SyncMessage>();
|
||||
while (true)
|
||||
{
|
||||
var msg = await NextAsync(reader, ct);
|
||||
all.Add(msg);
|
||||
if (msg.MsgCase == last)
|
||||
return all;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task SwallowAsync(Task task)
|
||||
{
|
||||
try { await task; }
|
||||
@@ -154,6 +189,12 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
return r[0];
|
||||
}
|
||||
|
||||
/// <summary>The tables individually flagged as owing the peer a snapshot (schema v3).</summary>
|
||||
private static async Task<List<string>> PendingSnapshotTables(SqliteLocalDb db) =>
|
||||
(await db.QueryAsync(
|
||||
"SELECT table_name FROM __localdb_snapshot_state WHERE needs_snapshot <> 0 ORDER BY table_name",
|
||||
x => x.GetString(0))).ToList();
|
||||
|
||||
private static async Task<long> OplogCount(SqliteLocalDb db)
|
||||
{
|
||||
var r = await db.QueryAsync("SELECT COUNT(*) FROM __localdb_oplog", x => x.GetInt64(0));
|
||||
@@ -170,6 +211,50 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
IsTombstone = false,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task MutualSnapshot_OverABoundedTransport_ConvergesWithoutDeadlock()
|
||||
{
|
||||
// BOTH sides owe a snapshot — the ordinary outcome of enabling replication late on a pair,
|
||||
// since baselining flags needs_snapshot on whichever node baselined and both nodes do.
|
||||
//
|
||||
// The snapshot used to be streamed from INSIDE the handshake, before the receive loop
|
||||
// existed, so neither side drained its inbox while sending. That exchange completed only
|
||||
// because the inbox was unbounded and could hold the peer's ENTIRE snapshot in memory —
|
||||
// which is the memory profile this design gave up on. Over a transport that pushes back
|
||||
// (which the gRPC duplex now is, with its bounded inbox) the old shape deadlocks outright.
|
||||
// Streaming the snapshot alongside an already-running receive loop is what makes both the
|
||||
// bound and this test possible.
|
||||
var options = new ReplicationOptions { MaxBatchSize = 2, FlushInterval = TimeSpan.FromMilliseconds(20) };
|
||||
var a = await NewSideAsync(options);
|
||||
var b = await NewSideAsync(options);
|
||||
|
||||
const int rowsPerSide = 60;
|
||||
for (var i = 1; i <= rowsPerSide; i++)
|
||||
{
|
||||
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
|
||||
await b.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'B', @id)", new { id = 100 + i });
|
||||
}
|
||||
await a.Store.SetNeedsSnapshotAsync(true, default);
|
||||
await b.Store.SetNeedsSnapshotAsync(true, default);
|
||||
|
||||
// Capacity 1 each way: the tightest possible bound, so a side that stops draining its inbox
|
||||
// to send stalls at once instead of getting away with it on buffer slack.
|
||||
var (da, db) = DuplexPair(capacity: 1);
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var runA = a.Session.RunAsync(da, cts.Token);
|
||||
var runB = b.Session.RunAsync(db, cts.Token);
|
||||
|
||||
await WaitForAsync(
|
||||
async () => (await ReadOrders(a.Db)).Count == 2 * rowsPerSide
|
||||
&& (await ReadOrders(b.Db)).Count == 2 * rowsPerSide,
|
||||
RunTimeout);
|
||||
Assert.Equal(await ReadOrders(a.Db), await ReadOrders(b.Db));
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(runA);
|
||||
await SwallowAsync(runB);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WipedPeer_AfterEverythingWasAckedAndPruned_StillGetsSnapshot()
|
||||
{
|
||||
@@ -292,6 +377,93 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
await SwallowAsync(runB);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CapPrune_IntoALiveSessionsUnsentRange_FaultsIt_AndTheReconnectSnapshots()
|
||||
{
|
||||
// P4-1. The pump reads `seq > sentThruSeq`, which resumes SILENTLY above a hole a cap prune
|
||||
// punched — so it keeps shipping later deltas, the peer acks them, and the pair is durably
|
||||
// divergent. `needs_snapshot` (set in the prune's own transaction) heals only a RECONNECT,
|
||||
// and nothing forced one: on a stable LAN with a slow-but-alive peer the session, and the
|
||||
// divergence, could last for days. Faulting is what turns the persisted flag into a snapshot.
|
||||
var a = await NewSideAsync();
|
||||
for (var i = 1; i <= 6; i++)
|
||||
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
|
||||
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
// The peer holds everything so far: no snapshot is owed, and the pump idles at seq 6.
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token, lastAppliedRemoteSeq: 6);
|
||||
|
||||
// Write three more rows and prune all but the newest, in ONE transaction — which is what a
|
||||
// cap prune is, and what makes this deterministic: the pump's reader can only ever see the
|
||||
// before state (nothing above seq 6) or the after state (seqs 7 and 8 already gone). It can
|
||||
// never catch rows 7/8 in between and legitimately advance past them.
|
||||
await using (var tx = await a.Db.BeginTransactionAsync(cts.Token))
|
||||
{
|
||||
await tx.ExecuteAsync(
|
||||
"INSERT INTO orders (id, sku, qty) VALUES (7,'PRUNED',7),(8,'PRUNED',8),(9,'KEPT',9)", null, cts.Token);
|
||||
await tx.ExecuteAsync("UPDATE __localdb_peer_state SET needs_snapshot = 1 WHERE id = 1", null, cts.Token);
|
||||
await tx.ExecuteAsync(
|
||||
"DELETE FROM __localdb_oplog WHERE seq < (SELECT MAX(seq) FROM __localdb_oplog)", null, cts.Token);
|
||||
await tx.CommitAsync(cts.Token);
|
||||
}
|
||||
|
||||
var fault = await Assert.ThrowsAsync<LocalDbOplogGapException>(() => run);
|
||||
Assert.Contains("6", fault.Message, StringComparison.Ordinal);
|
||||
|
||||
// And it faulted BEFORE shipping anything from above the hole: a delta for seq 9 would have
|
||||
// moved the peer's watermark past rows 7 and 8, which is the divergence itself.
|
||||
while (fromSession.TryRead(out var stray))
|
||||
Assert.NotEqual(SyncMessage.MsgOneofCase.DeltaBatch, stray.MsgCase);
|
||||
|
||||
// The reconnect is what heals: the prune's needs_snapshot makes the handshake owe a full
|
||||
// snapshot, which carries the pruned rows' current state to a real peer.
|
||||
var a2 = NewSessionOver(a.Db);
|
||||
var b = await NewSideAsync();
|
||||
var (da, db) = DuplexPair();
|
||||
var run2A = a2.Session.RunAsync(da, cts.Token);
|
||||
var run2B = b.Session.RunAsync(db, cts.Token);
|
||||
|
||||
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 9, RunTimeout);
|
||||
Assert.Equal(await ReadOrders(a.Db), await ReadOrders(b.Db));
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run2A);
|
||||
await SwallowAsync(run2B);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AckPruningToEmpty_DoesNotFaultALiveSession()
|
||||
{
|
||||
// The other half of the horizon check: ordinary ack-pruning deletes seq <= last_acked_seq,
|
||||
// which by construction sits at or below the sent watermark, so a converged pair that has
|
||||
// pruned its whole oplog must keep streaming. A gap test written against `needs_snapshot`,
|
||||
// or one that treated any risen horizon as a fault, would kill healthy sessions here.
|
||||
var a = await NewSideAsync();
|
||||
var b = await NewSideAsync();
|
||||
for (var i = 1; i <= 4; i++)
|
||||
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
|
||||
|
||||
var (da, db) = DuplexPair();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var runA = a.Session.RunAsync(da, cts.Token);
|
||||
var runB = b.Session.RunAsync(db, cts.Token);
|
||||
|
||||
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 4, RunTimeout);
|
||||
await WaitForAsync(async () => await OplogCount(a.Db) == 0, RunTimeout);
|
||||
|
||||
// Still live, still incremental: a post-prune write reaches B as an ordinary delta.
|
||||
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (5, 'POST', 5)");
|
||||
await WaitForAsync(async () => (await ReadOrders(b.Db)).Any(r => r.Id == 5), RunTimeout);
|
||||
Assert.False(runA.IsFaulted, runA.Exception?.ToString());
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(runA);
|
||||
await SwallowAsync(runB);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Snapshot_MergesNotTruncates()
|
||||
{
|
||||
@@ -536,6 +708,261 @@ public sealed class SnapshotResyncTests : IDisposable
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
// ---- Per-table snapshot scoping (0.3.0) ----------------------------------------------
|
||||
|
||||
private const string WidgetsSql = "CREATE TABLE widgets (id INTEGER PRIMARY KEY, name TEXT)";
|
||||
|
||||
/// <summary>
|
||||
/// A node with two registered tables where exactly one — <c>orders</c> — was baselined.
|
||||
/// <c>widgets</c> is registered BEFORE its rows are written, so ordinary capture ledgers them;
|
||||
/// <c>orders</c> is registered after, so only baselining can. That asymmetry is what makes the
|
||||
/// scope observable: a full snapshot carries both tables' ledgers, a scoped one carries orders
|
||||
/// alone — and both tables hold rows either way, so an empty widgets stream cannot be mistaken
|
||||
/// for "there was nothing to send".
|
||||
/// </summary>
|
||||
private async Task<Side> NewBaselinedSideAsync(ReplicationOptions? options = null)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
||||
_paths.Add(path);
|
||||
var db = new SqliteLocalDb(new LocalDbOptions { Path = path });
|
||||
_disposables.Add(db);
|
||||
await db.ExecuteAsync(OrdersSql);
|
||||
await db.ExecuteAsync(WidgetsSql);
|
||||
|
||||
// Predates registration: invisible to CDC, so only baselining puts it in the ledger.
|
||||
await db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (1,'PRE',1),(2,'PRE',2)");
|
||||
|
||||
db.RegisterReplicated("widgets");
|
||||
await db.ExecuteAsync("INSERT INTO widgets (id, name) VALUES (1,'W1'),(2,'W2')");
|
||||
|
||||
db.RegisterReplicated("orders", baselineExistingRows: true);
|
||||
|
||||
return NewSessionOver(db, options);
|
||||
}
|
||||
|
||||
/// <summary>The peer of <see cref="NewBaselinedSideAsync"/>: the same two tables, empty, owing nothing.</summary>
|
||||
private async Task<Side> NewTwoTableSideAsync(ReplicationOptions? options = null)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
||||
_paths.Add(path);
|
||||
var db = new SqliteLocalDb(new LocalDbOptions { Path = path });
|
||||
_disposables.Add(db);
|
||||
await db.ExecuteAsync(OrdersSql);
|
||||
await db.ExecuteAsync(WidgetsSql);
|
||||
db.RegisterReplicated("orders");
|
||||
db.RegisterReplicated("widgets");
|
||||
return NewSessionOver(db, options);
|
||||
}
|
||||
|
||||
private static List<string> SnapshotTablesIn(IEnumerable<SyncMessage> messages) =>
|
||||
messages.Where(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBatch)
|
||||
.Select(m => m.SnapshotBatch.TableName)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.Order(StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
[Fact]
|
||||
public async Task Baselining_OneTable_StreamsOnlyThatTable()
|
||||
{
|
||||
// The whole point of the per-table flag. Through 0.2.x the flag was one bit for the whole
|
||||
// database, so baselining `orders` re-streamed `widgets` (and everything else) in full — in
|
||||
// both directions at once on a pair enabling replication late, which on a database holding
|
||||
// an event log at its retention cap is the largest transfer this library performs.
|
||||
var a = await NewBaselinedSideAsync();
|
||||
Assert.Equal(["orders"], await PendingSnapshotTables(a.Db));
|
||||
Assert.Equal(0, await NeedsSnapshot(a.Db)); // NOT the database-wide flag
|
||||
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token, supportsPartialSnapshot: true);
|
||||
var frames = await DrainThroughAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
|
||||
|
||||
var begin = frames.Single(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBegin).SnapshotBegin;
|
||||
Assert.Equal(["orders"], begin.Tables);
|
||||
Assert.Equal(["orders"], SnapshotTablesIn(frames));
|
||||
// The baselined rows really are in there — the scope narrowed the snapshot, it did not empty it.
|
||||
var pks = frames.Where(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBatch)
|
||||
.SelectMany(m => m.SnapshotBatch.Rows).Select(r => r.PkJson).Order(StringComparer.Ordinal);
|
||||
Assert.Equal(["{\"id\":1}", "{\"id\":2}"], pks);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Baselining_AgainstAPeerWithoutPartialSupport_StreamsEveryTable()
|
||||
{
|
||||
// Mixed-version interop, the hard constraint. A pre-0.3.0 peer ignores SnapshotBegin.tables
|
||||
// and would advance last_applied_remote_seq to as_of_seq on complete — which for a SCOPED
|
||||
// snapshot silently discards every delta below as_of for the tables it left out. There is no
|
||||
// way to express "partial" to that peer, so it is served the 0.2.x answer: everything.
|
||||
var a = await NewBaselinedSideAsync();
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
// supportsPartialSnapshot omitted == what a 0.1.x/0.2.x handshake decodes to.
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
||||
var frames = await DrainThroughAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
|
||||
|
||||
var begin = frames.Single(m => m.MsgCase == SyncMessage.MsgOneofCase.SnapshotBegin).SnapshotBegin;
|
||||
Assert.Empty(begin.Tables); // empty == full, the only scope the old wire can express
|
||||
Assert.Equal(["orders", "widgets"], SnapshotTablesIn(frames));
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PartialSnapshot_DebtClearsOnlyWhenThePeerAcknowledges()
|
||||
{
|
||||
// Crash safety: an ungraceful exit part-way through a snapshot must re-stream it, so the
|
||||
// flag cannot come down when the frames are merely enqueued. The peer's SnapshotAck is
|
||||
// written after its own durable watermark write, so it means "I hold these rows".
|
||||
var a = await NewBaselinedSideAsync();
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token, supportsPartialSnapshot: true);
|
||||
var frames = await DrainThroughAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
|
||||
var complete = frames[^1].SnapshotComplete;
|
||||
|
||||
// The entire snapshot is on the wire and unacknowledged: the debt must still stand.
|
||||
Assert.Equal(["orders"], await PendingSnapshotTables(a.Db));
|
||||
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotAck = new SnapshotAck { AsOfSeq = complete.AsOfSeq } }, cts.Token);
|
||||
await WaitForAsync(async () => (await PendingSnapshotTables(a.Db)).Count == 0, RunTimeout);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Snapshot_ToAPeerWithoutPartialSupport_ClearsOnEnqueue()
|
||||
{
|
||||
// The 0.2.x behaviour, kept deliberately: a peer that sends no SnapshotAck would otherwise
|
||||
// leave the debt standing forever and re-stream the snapshot on every reconnect.
|
||||
var a = await NewBaselinedSideAsync();
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
||||
await DrainThroughAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotComplete, cts.Token);
|
||||
|
||||
await WaitForAsync(async () => (await PendingSnapshotTables(a.Db)).Count == 0, RunTimeout);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PartialSnapshot_DoesNotAdvanceTheReceiversWatermark()
|
||||
{
|
||||
// The receiver's half of the same invariant. last_applied_remote_seq is ONE number for the
|
||||
// whole peer oplog; moving it to a partial snapshot's as_of would declare the untouched
|
||||
// tables applied through that point, and their deltas below it would never be resent.
|
||||
var a = await NewSideAsync();
|
||||
await a.Store.SetLastAppliedRemoteSeqAsync(10, 0);
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token, supportsPartialSnapshot: true);
|
||||
|
||||
var begin = new SnapshotBegin { AsOfSeq = 900 };
|
||||
begin.Tables.Add("orders");
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotBegin = begin }, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage
|
||||
{
|
||||
SnapshotBatch = new SnapshotBatch { TableName = "orders", Rows = { SnapRow(1, "SNAP", 6_000_000) } },
|
||||
}, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = 900 } }, cts.Token);
|
||||
|
||||
// The ack proves the complete was processed — and that a 0.3.0 receiver answers one at all.
|
||||
var ack = await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.SnapshotAck, cts.Token);
|
||||
Assert.Equal(900, ack.SnapshotAck.AsOfSeq);
|
||||
|
||||
Assert.Equal(10, await LastAppliedRemoteSeq(a.Db)); // NOT 900
|
||||
Assert.Single(await ReadOrders(a.Db)); // the rows still merged
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FullSnapshot_ToAnOldPeer_DrawsNoSnapshotAck()
|
||||
{
|
||||
// Symmetry check on the negotiation: a 0.3.0 receiver must not put a message a pre-0.3.0
|
||||
// sender has never heard of on the wire back to it.
|
||||
var a = await NewSideAsync();
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var run = a.Session.RunAsync(duplex, cts.Token);
|
||||
|
||||
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotBegin = new SnapshotBegin { AsOfSeq = 300 } }, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage
|
||||
{
|
||||
SnapshotBatch = new SnapshotBatch { TableName = "orders", Rows = { SnapRow(1, "SNAP", 6_000_000) } },
|
||||
}, cts.Token);
|
||||
await toSession.WriteAsync(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = 300 } }, cts.Token);
|
||||
|
||||
// A full snapshot from an old peer still advances the watermark — that is the 0.2.x contract.
|
||||
await WaitForAsync(async () => await LastAppliedRemoteSeq(a.Db) == 300, RunTimeout);
|
||||
// Barrier: an ack for this delta proves everything before it was processed, so an absent
|
||||
// SnapshotAck is absent rather than merely late.
|
||||
await toSession.WriteAsync(new SyncMessage
|
||||
{
|
||||
DeltaBatch = new DeltaBatch { Entries = { new OplogEntry
|
||||
{
|
||||
Seq = 301, TableName = "orders", PkJson = "{\"id\":2}",
|
||||
RowJson = "{\"id\":2,\"sku\":\"BARRIER\",\"qty\":1}", Hlc = 6_000_001, NodeId = "peer", IsTombstone = false,
|
||||
} } },
|
||||
}, cts.Token);
|
||||
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.DeltaAck, cts.Token);
|
||||
Assert.False(fromSession.TryPeek(out var stray) && stray.MsgCase == SyncMessage.MsgOneofCase.SnapshotAck);
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BaselinedTable_ReachesARealPeer_WithoutRestreamingTheOthers()
|
||||
{
|
||||
// End to end over two real databases: the baselined table's pre-existing rows arrive via the
|
||||
// scoped snapshot, the untouched table's rows arrive as ordinary deltas, and the debt clears.
|
||||
var options = new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) };
|
||||
var a = await NewBaselinedSideAsync(options);
|
||||
var b = await NewTwoTableSideAsync(options);
|
||||
|
||||
var (da, db) = DuplexPair();
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var runA = a.Session.RunAsync(da, cts.Token);
|
||||
var runB = b.Session.RunAsync(db, cts.Token);
|
||||
|
||||
async Task<bool> Converged() =>
|
||||
(await ReadOrders(b.Db)).Count == 2
|
||||
&& (await b.Db.QueryAsync("SELECT COUNT(*) FROM widgets", x => x.GetInt64(0)))[0] == 2;
|
||||
|
||||
await WaitForAsync(Converged, RunTimeout);
|
||||
Assert.Equal(await ReadOrders(a.Db), await ReadOrders(b.Db));
|
||||
await WaitForAsync(async () => (await PendingSnapshotTables(a.Db)).Count == 0, RunTimeout);
|
||||
|
||||
// A partial snapshot speaks for nothing outside its tables, so neither side may skip the
|
||||
// deltas it covered: B's watermark tracks the oplog, not the snapshot's as-of.
|
||||
Assert.Equal(0, await NeedsSnapshot(a.Db));
|
||||
|
||||
cts.Cancel();
|
||||
await SwallowAsync(runA);
|
||||
await SwallowAsync(runB);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sender_ClearsNeedsSnapshotFlag()
|
||||
{
|
||||
|
||||
@@ -208,6 +208,50 @@ public sealed class SyncSessionTests : IDisposable
|
||||
Assert.Empty(await ReadOrders(b.Db));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handshake_FaultTeardown_StillTransmitsTheLocalHandshake()
|
||||
{
|
||||
// Schema-mismatch detection is symmetric: each side computes its verdict FROM the other's
|
||||
// Handshake. So a session that faults on the peer's handshake must still flush its own —
|
||||
// otherwise the healthy peer learns nothing and waits out its receive cap (tests) or
|
||||
// misclassifies the mismatch as a generic disconnect and reconnect-loops (production).
|
||||
//
|
||||
// The race being pinned: with the peer's handshake already buffered, the path from
|
||||
// enqueueing our Handshake on the control lane to the validation throw has no yields,
|
||||
// while the writer's wakeup sits in the thread-pool queue — a teardown that cancels the
|
||||
// writer can therefore kill the Handshake before it ever transmits. Preloading makes the
|
||||
// sprint as tight as possible, and running the scenarios concurrently starves the pool
|
||||
// the way a busy test suite does.
|
||||
const int Iterations = 40;
|
||||
var sides = new List<Side>(Iterations);
|
||||
for (var i = 0; i < Iterations; i++)
|
||||
sides.Add(await NewSideAsync());
|
||||
|
||||
using var cts = new CancellationTokenSource(RunTimeout);
|
||||
var runs = new List<(Task Run, ChannelReader<SyncMessage> FromSession)>(Iterations);
|
||||
foreach (var side in sides)
|
||||
{
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
var mismatched = MatchingHandshake(side.Db, "peer");
|
||||
mismatched.Tables[0].Digest = "not-the-local-digest";
|
||||
await toSession.WriteAsync(new SyncMessage { Handshake = mismatched }, cts.Token);
|
||||
runs.Add((side.Session.RunAsync(duplex, cts.Token), fromSession));
|
||||
}
|
||||
|
||||
var lost = 0;
|
||||
foreach (var (run, fromSession) in runs)
|
||||
{
|
||||
await Assert.ThrowsAsync<LocalDbSchemaMismatchException>(() => run);
|
||||
var sent = false;
|
||||
while (fromSession.TryRead(out var msg))
|
||||
sent |= msg.MsgCase == SyncMessage.MsgOneofCase.Handshake;
|
||||
if (!sent)
|
||||
lost++;
|
||||
}
|
||||
|
||||
Assert.True(lost == 0, $"{lost}/{Iterations} sessions tore down without transmitting their own Handshake");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handshake_SchemaVersionMismatch_FailsClosed()
|
||||
{
|
||||
@@ -553,8 +597,9 @@ public sealed class SyncSessionTests : IDisposable
|
||||
await a.Store.SetNeedsSnapshotAsync(true, default);
|
||||
var (duplex, toSession, fromSession) = ScriptedPeer();
|
||||
var sent = new TaskCompletionSource();
|
||||
a.Session.SnapshotSender = async (send, token) =>
|
||||
a.Session.SnapshotSender = async (scope, send, token) =>
|
||||
{
|
||||
Assert.False(scope.IsPartial); // the database-wide flag always means every table
|
||||
sent.TrySetResult();
|
||||
await send(new SyncMessage { SnapshotComplete = new SnapshotComplete { AsOfSeq = 7 } }, token);
|
||||
return 7;
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
using System.Threading.Channels;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.LocalDb.Contracts;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
|
||||
namespace ZB.MOM.WW.LocalDb.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The passive service's single-stream guard under a HALF-OPEN link: the dialling node is gone, but
|
||||
/// nothing on this side knows it, so the handler sits inside <c>MoveNext</c> forever holding the
|
||||
/// guard. Through 0.3.1 that made the pair unrecoverable without a restart — every reconnect from
|
||||
/// the rebooted peer was refused <c>AlreadyExists</c>, indefinitely on an idle pair, while this node
|
||||
/// went on reporting itself connected. The guard is there to serialize sessions, not to fence out a
|
||||
/// caller the auth interceptor already vouched for, so the newest dialler wins.
|
||||
/// </summary>
|
||||
public sealed class SyncStreamGuardTests : IDisposable
|
||||
{
|
||||
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10);
|
||||
|
||||
private readonly List<string> _paths = [];
|
||||
private readonly List<IDisposable> _disposables = [];
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var d in _disposables)
|
||||
d.Dispose();
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
foreach (var p in _paths)
|
||||
{
|
||||
if (File.Exists(p)) File.Delete(p);
|
||||
if (File.Exists(p + "-wal")) File.Delete(p + "-wal");
|
||||
if (File.Exists(p + "-shm")) File.Delete(p + "-shm");
|
||||
}
|
||||
}
|
||||
|
||||
private LocalDbSyncService NewService()
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
||||
_paths.Add(path);
|
||||
var db = new Internal.SqliteLocalDb(new LocalDbOptions { Path = path });
|
||||
_disposables.Add(db);
|
||||
db.ExecuteAsync("CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT)").GetAwaiter().GetResult();
|
||||
db.RegisterReplicated("orders");
|
||||
|
||||
return new LocalDbSyncService(
|
||||
db,
|
||||
Options.Create(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) }),
|
||||
NullLoggerFactory.Instance);
|
||||
}
|
||||
|
||||
private static ServerCallContext ContextFor(CancellationToken ct) => new FakeCallContext(ct);
|
||||
|
||||
[Fact]
|
||||
public async Task WedgedIncumbent_IsDisplacedByANewStream()
|
||||
{
|
||||
var service = NewService();
|
||||
// Eviction is normally generous; the test only needs it to not be the thing under test.
|
||||
service.TakeoverTimeout = Timeout;
|
||||
|
||||
// The incumbent's peer crashed: its stream will never deliver another message, and neither
|
||||
// end times out on silence — a converged pair is legitimately quiet. The handler parks in
|
||||
// the handshake's first read, holding the guard.
|
||||
using var incumbentCts = new CancellationTokenSource();
|
||||
var incumbentOut = new RecordingWriter();
|
||||
var incumbent = service.Sync(new SilentReader(), incumbentOut, ContextFor(incumbentCts.Token));
|
||||
|
||||
// It really is running and really did claim the guard (its handshake is on the wire).
|
||||
await incumbentOut.NextAsync(Timeout);
|
||||
Assert.False(incumbent.IsCompleted);
|
||||
|
||||
// The rebooted peer dials again. Pre-fix this threw AlreadyExists and the pair stayed wedged.
|
||||
using var newcomerCts = new CancellationTokenSource();
|
||||
var newcomerOut = new RecordingWriter();
|
||||
var newcomer = service.Sync(new SilentReader(), newcomerOut, ContextFor(newcomerCts.Token));
|
||||
|
||||
// The incumbent is torn down — gracefully, not as a fault of this node.
|
||||
await incumbent.WaitAsync(Timeout);
|
||||
// ...and the new stream is genuinely served: its own handshake goes out.
|
||||
var handshake = await newcomerOut.NextAsync(Timeout);
|
||||
Assert.Equal(SyncMessage.MsgOneofCase.Handshake, handshake.MsgCase);
|
||||
Assert.False(newcomer.IsCompleted);
|
||||
|
||||
newcomerCts.Cancel();
|
||||
await SwallowAsync(newcomer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DisplacedIncumbent_ReleasesTheGuard_SoAThirdStreamIsAlsoServed()
|
||||
{
|
||||
// Takeover must leave the guard in a state that can be taken over AGAIN — a handover that
|
||||
// leaked the slot would simply move the wedge one reconnect down the line.
|
||||
var service = NewService();
|
||||
service.TakeoverTimeout = Timeout;
|
||||
|
||||
using var firstCts = new CancellationTokenSource();
|
||||
var firstOut = new RecordingWriter();
|
||||
var first = service.Sync(new SilentReader(), firstOut, ContextFor(firstCts.Token));
|
||||
await firstOut.NextAsync(Timeout);
|
||||
|
||||
using var secondCts = new CancellationTokenSource();
|
||||
var secondOut = new RecordingWriter();
|
||||
var second = service.Sync(new SilentReader(), secondOut, ContextFor(secondCts.Token));
|
||||
await first.WaitAsync(Timeout);
|
||||
await secondOut.NextAsync(Timeout);
|
||||
|
||||
using var thirdCts = new CancellationTokenSource();
|
||||
var thirdOut = new RecordingWriter();
|
||||
var third = service.Sync(new SilentReader(), thirdOut, ContextFor(thirdCts.Token));
|
||||
await second.WaitAsync(Timeout);
|
||||
Assert.Equal(SyncMessage.MsgOneofCase.Handshake, (await thirdOut.NextAsync(Timeout)).MsgCase);
|
||||
|
||||
thirdCts.Cancel();
|
||||
await SwallowAsync(third);
|
||||
}
|
||||
|
||||
private static async Task SwallowAsync(Task task)
|
||||
{
|
||||
try { await task.WaitAsync(Timeout); }
|
||||
catch (OperationCanceledException) { }
|
||||
catch (RpcException) { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The minimum <see cref="ServerCallContext"/> the handler actually reads: its cancellation
|
||||
/// token. (Grpc.Core.Api ships no test double, and the ASP.NET Core one is not reachable outside
|
||||
/// a live call.)
|
||||
/// </summary>
|
||||
private sealed class FakeCallContext(CancellationToken ct) : ServerCallContext
|
||||
{
|
||||
protected override string MethodCore => "/localdb_sync.v1.LocalDbSync/Sync";
|
||||
protected override string HostCore => "test";
|
||||
protected override string PeerCore => "test";
|
||||
protected override DateTime DeadlineCore => DateTime.MaxValue;
|
||||
protected override Metadata RequestHeadersCore => [];
|
||||
protected override CancellationToken CancellationTokenCore => ct;
|
||||
protected override Metadata ResponseTrailersCore { get; } = [];
|
||||
protected override Status StatusCore { get; set; }
|
||||
protected override WriteOptions? WriteOptionsCore { get; set; }
|
||||
protected override AuthContext AuthContextCore => new(null, new Dictionary<string, List<AuthProperty>>());
|
||||
|
||||
protected override ContextPropagationToken CreatePropagationTokenCore(ContextPropagationOptions? options) =>
|
||||
throw new NotSupportedException();
|
||||
|
||||
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>A stream that never delivers anything and never ends — a half-open link.</summary>
|
||||
private sealed class SilentReader : IAsyncStreamReader<SyncMessage>
|
||||
{
|
||||
public SyncMessage Current => throw new InvalidOperationException("no message");
|
||||
|
||||
public async Task<bool> MoveNext(CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class RecordingWriter : IServerStreamWriter<SyncMessage>
|
||||
{
|
||||
private readonly Channel<SyncMessage> _written = Channel.CreateUnbounded<SyncMessage>();
|
||||
|
||||
public WriteOptions? WriteOptions { get; set; }
|
||||
|
||||
public Task WriteAsync(SyncMessage message) => WriteAsync(message, CancellationToken.None);
|
||||
|
||||
public Task WriteAsync(SyncMessage message, CancellationToken cancellationToken)
|
||||
{
|
||||
_written.Writer.TryWrite(message);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task<SyncMessage> NextAsync(TimeSpan timeout)
|
||||
{
|
||||
using var cts = new CancellationTokenSource(timeout);
|
||||
return await _written.Reader.ReadAsync(cts.Token);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
# Host-side publish output copied into the runtime-only Docker image.
|
||||
docker/publish/
|
||||
|
||||
# Serilog rolling file sink, written relative to the content root.
|
||||
logs/
|
||||
@@ -0,0 +1,32 @@
|
||||
<Project>
|
||||
|
||||
<!--
|
||||
Workspace-wide build settings for ZB.MOM.WW.Overview (the family overview dashboard).
|
||||
Mirrors the HistorianGateway workspace props — the reference app layout for this family.
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<LangVersion>latest</LangVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<!--
|
||||
Build-quality floor: the family's zero-warnings invariant is a build GATE, not a convention —
|
||||
new warnings break the build and are fixed rather than suppressed.
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<AnalysisLevel>latest</AnalysisLevel>
|
||||
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||
<Deterministic>true</Deterministic>
|
||||
</PropertyGroup>
|
||||
|
||||
<!--
|
||||
This is an APP, not a library: nothing here is ever packed to the Gitea feed.
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,8 @@
|
||||
<Solution>
|
||||
<Folder Name="/src/">
|
||||
<Project Path="src/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/tests/">
|
||||
<Project Path="tests/ZB.MOM.WW.Overview.Tests/ZB.MOM.WW.Overview.Tests.csproj" />
|
||||
</Folder>
|
||||
</Solution>
|
||||
@@ -0,0 +1,2 @@
|
||||
**/*.md
|
||||
**/.DS_Store
|
||||
@@ -0,0 +1,29 @@
|
||||
# Runtime-only image for the ZB.MOM.WW.Overview dashboard.
|
||||
#
|
||||
# The app is published FRAMEWORK-DEPENDENT on the host (which holds the Gitea NuGet feed
|
||||
# credentials in ~/.nuget) into ./publish, then copied in here. Docker performs NO restore and
|
||||
# NO build, so the authenticated Gitea feed is never needed at image-build time. Regenerate
|
||||
# ./publish with:
|
||||
# dotnet publish src/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.csproj \
|
||||
# -c Release -o docker/publish -p:UseAppHost=false
|
||||
#
|
||||
# The published IL is architecture-neutral, so the multi-arch aspnet:10.0 base runs as-is on
|
||||
# both Apple Silicon and amd64 hosts.
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime
|
||||
WORKDIR /app
|
||||
|
||||
# The base image declares its own binding — as ASPNETCORE_HTTP_PORTS=8080 on .NET 8+, NOT the
|
||||
# ASPNETCORE_URLS everyone reaches for first (clearing only URLS leaves the warning in place, which
|
||||
# is a good way to spend an afternoon). Left set, it collides with the Kestrel endpoint section in
|
||||
# appsettings.Docker.json: Kestrel wins, but logs "Overriding address(es) 'http://*:8080'" on every
|
||||
# boot — the family's URLs-override trap arriving by default rather than by mistake. Clearing both
|
||||
# at the layer that introduced them leaves the Kestrel section as the single authority.
|
||||
ENV ASPNETCORE_URLS= \
|
||||
ASPNETCORE_HTTP_PORTS=
|
||||
|
||||
COPY publish/ ./
|
||||
|
||||
# Dashboard + /health/* + /metrics, all HTTP/1.1 and all anonymous.
|
||||
EXPOSE 5320
|
||||
|
||||
ENTRYPOINT ["dotnet", "ZB.MOM.WW.Overview.dll"]
|
||||
@@ -0,0 +1,81 @@
|
||||
# `docker/` — containerised overview dashboard
|
||||
|
||||
Runtime-only image, the HistorianGateway pattern: publish on the host (which has the Gitea feed
|
||||
credentials), copy the output into `mcr.microsoft.com/dotnet/aspnet:10.0`. Docker never restores.
|
||||
|
||||
```bash
|
||||
# from ZB.MOM.WW.Overview/
|
||||
dotnet publish src/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.csproj \
|
||||
-c Release -o docker/publish -p:UseAppHost=false
|
||||
|
||||
cd docker && docker compose up -d --build
|
||||
open http://localhost:5320
|
||||
```
|
||||
|
||||
`docker/publish/` is build output and is gitignored.
|
||||
|
||||
## Why the container, and not just `dotnet run`
|
||||
|
||||
The dev rigs publish only some node ports to the host:
|
||||
|
||||
| Fleet | Health endpoint | Reachable from the Mac host? |
|
||||
|---|---|---|
|
||||
| ScadaBridge central-a/b | container `:5000` | yes — `localhost:9001` / `:9002` |
|
||||
| ScadaBridge site-\* (6) | container `:8084` | **no** — not published |
|
||||
| OtOpcUa central-1/2 | container `:9000` | **no** — only Traefik `:9200`, load-balanced across the pair |
|
||||
| OtOpcUa site-\* (4) | container `:8080` | **no** — not published |
|
||||
| HistorianGateway | container `:5220` | yes — `localhost:5220` |
|
||||
| MxAccessGateway | windev `:5130` | yes, with the VPN up |
|
||||
|
||||
So a host-run dashboard can only see part of the fleet. This stack joins the three rig networks
|
||||
(`otopcua-dev_default`, `scadabridge-net`, `zb-historiangw_default`, all declared `external`) and
|
||||
addresses every node by container DNS name, which is how it sees all of them.
|
||||
|
||||
Ports above were established by probing the running rigs on 2026-07-24, **not** read off the
|
||||
compose files — the two disagree. In particular OtOpcUa's site nodes serve health on `:8080`, not
|
||||
the `:9000` their compose anchor implies, because their explicit Kestrel listeners for LocalDb sync
|
||||
re-bind the primary HTTP port.
|
||||
|
||||
Start the rigs before this stack. The networks are `external`, so bringing the dashboard up or
|
||||
down never disturbs a running rig.
|
||||
|
||||
## Registry shape
|
||||
|
||||
Bound from the `Overview` section; `appsettings.Docker.json` is the worked example.
|
||||
|
||||
```jsonc
|
||||
"Overview": {
|
||||
"PollIntervalSeconds": 10, // sweep cadence; per-app and per-instance overrides allowed
|
||||
"TimeoutSeconds": 3, // per-request timeout
|
||||
"StaleAfterSeconds": 45, // must exceed the poll interval, or every card reads Stale
|
||||
"Applications": [
|
||||
{
|
||||
"Name": "ScadaBridge",
|
||||
"ManagementLabel": "CentralUI", // label on each instance's management link
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "central-a",
|
||||
"Group": "central", // one Akka cluster = one group; leader + split-brain
|
||||
// detection are scoped to it. Omit for standalone apps.
|
||||
"BaseUrl": "http://host:5000", // /health/ready and /health/active are probed under this
|
||||
"ManagementUrl": "http://localhost:9001/", // followed by the OPERATOR'S browser, so it
|
||||
// must be host-reachable, not container DNS
|
||||
"HasActiveRole": true // false for single-instance gateways: no Active badge,
|
||||
// and /health/active is never probed
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`appsettings.json` ships the tuning defaults with an **empty** `Applications` list, so a deployment
|
||||
that supplies no registry fails at startup naming the missing key rather than booting into an empty
|
||||
dashboard.
|
||||
|
||||
## Configuration traps
|
||||
|
||||
- **Never set `ASPNETCORE_URLS` alongside `Kestrel__Endpoints__*`.** Declaring both puts Kestrel
|
||||
into explicit-endpoints mode and one of them is silently discarded.
|
||||
- No `env_file`, no secrets, no login. The dashboard reads anonymous health endpoints and holds no
|
||||
credentials. That is a requirement of the design, not an omission.
|
||||
@@ -0,0 +1,54 @@
|
||||
# Local Docker deployment of the SCADA family overview dashboard.
|
||||
#
|
||||
# Why this exists rather than just `dotnet run`: the dev rigs publish only some node ports to the
|
||||
# host. OtOpcUa publishes no per-node HTTP port at all (only Traefik's load-balanced :9200, which
|
||||
# round-robins the central pair and so cannot identify a node), and ScadaBridge's site nodes keep
|
||||
# their health port container-internal. A dashboard on the host can therefore see a fraction of the
|
||||
# fleet; one joined to the rig networks sees all of it, addressing each node by container DNS name.
|
||||
#
|
||||
# Registry: appsettings.Docker.json, baked into the image (ASPNETCORE_ENVIRONMENT=Docker below).
|
||||
#
|
||||
# No env_file and no secrets: the dashboard reads anonymous /health endpoints, holds no
|
||||
# credentials, and serves no login. That is a requirement, not an oversight.
|
||||
#
|
||||
# Explicit project name so this stack is isolated from the other compose projects on the host.
|
||||
name: zb-overview
|
||||
|
||||
services:
|
||||
overview:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: zb-overview:local
|
||||
container_name: zb-overview
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "5320:5320" # dashboard + /health/* + /metrics
|
||||
environment:
|
||||
ASPNETCORE_ENVIRONMENT: Docker
|
||||
|
||||
# The endpoint lives in appsettings.Docker.json — ONE authority for the binding. Do NOT add
|
||||
# ASPNETCORE_URLS here; the Dockerfile clears the base image's default for that reason.
|
||||
|
||||
# No container healthcheck on purpose. aspnet:10.0 ships neither curl nor wget, and adding a
|
||||
# `--healthcheck` mode to the app to work around that would put a second entry point into
|
||||
# production code purely to satisfy Compose. The app's real health surface is HTTP and already
|
||||
# honest — `curl localhost:5320/health/ready` from the host, or scrape /metrics.
|
||||
|
||||
networks:
|
||||
- otopcua
|
||||
- scadabridge
|
||||
- historiangw
|
||||
|
||||
# All three are created by the rigs themselves; this stack joins them and never defines them, so
|
||||
# bringing the dashboard up or down cannot disturb a running rig. Start the rigs first.
|
||||
networks:
|
||||
otopcua:
|
||||
name: otopcua-dev_default
|
||||
external: true
|
||||
scadabridge:
|
||||
name: scadabridge-net
|
||||
external: true
|
||||
historiangw:
|
||||
name: zb-historiangw_default
|
||||
external: true
|
||||
@@ -0,0 +1,24 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<configuration>
|
||||
<packageSources>
|
||||
<clear />
|
||||
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
|
||||
<add key="dohertj2-gitea" value="https://gitea.dohertylan.com/api/packages/dohertj2/nuget/index.json" />
|
||||
</packageSources>
|
||||
<!-- nuget.org serves everything; the Gitea feed serves only the ZB.MOM.WW.* shared libs.
|
||||
Credentials are NOT committed: they are provided per-developer at the user level
|
||||
(~/.nuget/NuGet/NuGet.Config). -->
|
||||
<packageSourceMapping>
|
||||
<packageSource key="nuget.org">
|
||||
<package pattern="*" />
|
||||
</packageSource>
|
||||
<packageSource key="dohertj2-gitea">
|
||||
<package pattern="ZB.MOM.WW.Health" />
|
||||
<package pattern="ZB.MOM.WW.Health.*" />
|
||||
<package pattern="ZB.MOM.WW.Telemetry" />
|
||||
<package pattern="ZB.MOM.WW.Telemetry.*" />
|
||||
<package pattern="ZB.MOM.WW.Configuration" />
|
||||
<package pattern="ZB.MOM.WW.Theme" />
|
||||
</packageSource>
|
||||
</packageSourceMapping>
|
||||
</configuration>
|
||||
@@ -0,0 +1,22 @@
|
||||
@* Host page. Head order matters: Bootstrap first (the Theme kit's shell and
|
||||
TechButton are built on Bootstrap utility classes), then <ThemeHead/> which
|
||||
brings the design tokens and IBM Plex, then site.css so this app's own
|
||||
dashboard anatomy can override either. *@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<base href="/" />
|
||||
<link rel="stylesheet" href="/lib/bootstrap/css/bootstrap.min.css" />
|
||||
<ThemeHead />
|
||||
<link rel="stylesheet" href="/css/site.css" />
|
||||
<HeadOutlet @rendermode="InteractiveServer" />
|
||||
</head>
|
||||
<body>
|
||||
<Routes @rendermode="InteractiveServer" />
|
||||
<script src="/lib/bootstrap/js/bootstrap.bundle.min.js"></script>
|
||||
<ThemeScripts />
|
||||
<script src="/_framework/blazor.web.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,43 @@
|
||||
@inherits LayoutComponentBase
|
||||
@inject OverviewSnapshotStore Store
|
||||
|
||||
@* Thin layout: the side-rail chassis belongs to the shared kit's <ThemeShell>.
|
||||
No AuthorizeView anywhere — this dashboard is anonymous by requirement, so
|
||||
there is no user to show and nothing to sign out of. The rail's application
|
||||
links are fragment anchors into the single page rather than routes: the whole
|
||||
fleet is one view, and splitting it per application would defeat the point. *@
|
||||
<ThemeShell Product="SCADA Overview" Accent="#6d28d9">
|
||||
<Nav>
|
||||
<NavRailItem Href="/" Text="Overview" Match="NavLinkMatch.All" />
|
||||
@if (Applications.Count > 0)
|
||||
{
|
||||
<NavRailSection Title="Applications" Key="overview-apps">
|
||||
@foreach (var application in Applications)
|
||||
{
|
||||
<a class="rail-link" href="@($"#{Slug(application)}")">@application</a>
|
||||
}
|
||||
</NavRailSection>
|
||||
}
|
||||
</Nav>
|
||||
<RailFooter>
|
||||
<div class="rail-note">read-only · anonymous<br />ZB.MOM.WW.Overview @Version</div>
|
||||
</RailFooter>
|
||||
<ChildContent>@Body</ChildContent>
|
||||
</ThemeShell>
|
||||
|
||||
@code {
|
||||
private static readonly string Version =
|
||||
typeof(MainLayout).Assembly.GetName().Version?.ToString(3) ?? "0.0.0";
|
||||
|
||||
private IReadOnlyList<string> Applications =>
|
||||
Store.Current.Applications.Select(a => a.Name).ToList();
|
||||
|
||||
/// <summary>
|
||||
/// Builds the fragment id for an application section. Must stay identical to the id the
|
||||
/// Overview page emits, or every rail link becomes a dead anchor.
|
||||
/// </summary>
|
||||
/// <param name="name">The application's display name.</param>
|
||||
/// <returns>A URL-fragment-safe slug.</returns>
|
||||
internal static string Slug(string name) =>
|
||||
new(name.Where(c => char.IsLetterOrDigit(c) || c is '-' or '_').Select(char.ToLowerInvariant).ToArray());
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
@page "/"
|
||||
@implements IDisposable
|
||||
@inject OverviewSnapshotStore Store
|
||||
@inject TimeProvider Time
|
||||
|
||||
@* The whole dashboard, rendered from the snapshot store. This page never polls
|
||||
an instance: it subscribes to the store and re-renders when the poller swaps
|
||||
a snapshot in, plus a slow local tick so ages and staleness advance on their
|
||||
own between sweeps. *@
|
||||
|
||||
@* Read once per render and threaded through everything below, so every card, KPI
|
||||
and group header on one paint agrees on what "now" is. Reading the clock at
|
||||
render — rather than caching it when a snapshot arrives — is what lets the view
|
||||
age on its own: a paused or stuck poller publishes nothing, and the cards still
|
||||
cross into Stale. *@
|
||||
@{
|
||||
var now = Time.GetUtcNow();
|
||||
}
|
||||
|
||||
<PageTitle>SCADA Overview</PageTitle>
|
||||
|
||||
<div class="page-head rise">
|
||||
<h1>Fleet Overview</h1>
|
||||
<span class="page-meta">@Meta</span>
|
||||
<span class="spacer"></span>
|
||||
<span class="conn-pill" data-state="@(_paused ? "disconnected" : "connected")">
|
||||
<span class="dot"></span>@(_paused ? "Paused" : "Polling")
|
||||
</span>
|
||||
<button class="rail-btn" type="button" data-testid="pause-toggle" @onclick="TogglePause">
|
||||
@(_paused ? "Resume auto-refresh" : "Pause auto-refresh")
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@if (_snapshot.Applications.Count == 0)
|
||||
{
|
||||
<p class="page-meta" data-testid="empty-registry">
|
||||
No instances have been polled yet.
|
||||
</p>
|
||||
}
|
||||
else
|
||||
{
|
||||
<div class="agg-grid rise">
|
||||
<div class="agg-card">
|
||||
<div class="agg-label">Instances</div>
|
||||
<div class="agg-value">@_instances.Count<span class="agg-sub">registered</span></div>
|
||||
</div>
|
||||
<div class="agg-card good">
|
||||
<div class="agg-label">Up</div>
|
||||
<div class="agg-value" data-testid="kpi-up">@Count(now, InstanceStatus.Up)</div>
|
||||
</div>
|
||||
<div class="agg-card @(Count(now, InstanceStatus.Degraded) > 0 ? "caution" : null)">
|
||||
<div class="agg-label">Degraded</div>
|
||||
<div class="agg-value" data-testid="kpi-degraded">@Count(now, InstanceStatus.Degraded)</div>
|
||||
</div>
|
||||
<div class="agg-card @(Count(now, InstanceStatus.Down) > 0 ? "alert" : null)">
|
||||
<div class="agg-label">Down</div>
|
||||
<div class="agg-value" data-testid="kpi-down">@Count(now, InstanceStatus.Down)</div>
|
||||
</div>
|
||||
<div class="agg-card @(Count(now, InstanceStatus.Unreachable) > 0 ? "alert" : null)">
|
||||
<div class="agg-label">Unreachable</div>
|
||||
<div class="agg-value" data-testid="kpi-unreachable">@Count(now, InstanceStatus.Unreachable)</div>
|
||||
</div>
|
||||
<div class="agg-card quiet">
|
||||
<div class="agg-label">Stale</div>
|
||||
<div class="agg-value" data-testid="kpi-stale">@StaleCount(now)</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@foreach (var application in _snapshot.Applications)
|
||||
{
|
||||
<section class="app-section rise" id="@MainLayout.Slug(application.Name)" data-testid="application">
|
||||
<div class="app-head">
|
||||
<h2>@application.Name</h2>
|
||||
<StatusPill State="RollupState(application)">@RollupLabel(application)</StatusPill>
|
||||
<span class="spacer"></span>
|
||||
<span class="app-meta">@AppMeta(application)</span>
|
||||
</div>
|
||||
|
||||
@foreach (var group in Layout(application))
|
||||
{
|
||||
<div class="group">
|
||||
@if (group.Key is { } groupName)
|
||||
{
|
||||
var state = application.GroupFor(groupName);
|
||||
<div class="group-head">
|
||||
<span class="group-name">@groupName</span>
|
||||
@if (state?.LeaderInstance is { } leaderInstance)
|
||||
{
|
||||
<StatusPill State="StatusState.Info">Leader · @leaderInstance</StatusPill>
|
||||
}
|
||||
else if (state?.LeaderAddress is { } leaderAddress)
|
||||
{
|
||||
@* Unresolved address: shown raw rather than hidden — a leader on a
|
||||
host that is not in the registry is itself the finding. *@
|
||||
<StatusPill State="StatusState.Info">Leader · @leaderAddress</StatusPill>
|
||||
}
|
||||
@if (state?.Disagreement == true)
|
||||
{
|
||||
<StatusPill State="StatusState.Bad">
|
||||
Split brain · @string.Join(" vs ", state.ReportedLeaders)
|
||||
</StatusPill>
|
||||
}
|
||||
<span class="group-meta">@GroupMeta(group, now)</span>
|
||||
</div>
|
||||
}
|
||||
<div class="card-grid">
|
||||
@foreach (var instance in group)
|
||||
{
|
||||
<InstanceCard Instance="instance"
|
||||
Now="now"
|
||||
IsLeader="IsLeader(application, instance)" />
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</section>
|
||||
}
|
||||
}
|
||||
|
||||
@code {
|
||||
/// <summary>
|
||||
/// How often the page re-renders on its own. Independent of the poll interval: its job is to
|
||||
/// age the "last poll" figures and let a card cross into Stale even when no sweep lands —
|
||||
/// which is exactly the case a stuck poller produces.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan UiTick = TimeSpan.FromSeconds(5);
|
||||
|
||||
private OverviewSnapshot _snapshot = OverviewSnapshot.Empty;
|
||||
private IReadOnlyList<InstanceSnapshot> _instances = [];
|
||||
private bool _paused;
|
||||
private ITimer? _timer;
|
||||
|
||||
protected override void OnInitialized()
|
||||
{
|
||||
Adopt(Store.Current);
|
||||
Store.Changed += OnSnapshotChanged;
|
||||
_timer = Time.CreateTimer(_ => InvokeAsync(StateHasChanged), null, UiTick, UiTick);
|
||||
}
|
||||
|
||||
private void OnSnapshotChanged(OverviewSnapshot snapshot)
|
||||
{
|
||||
// Pause stops ADOPTING new snapshots; the poller keeps sweeping. That is what makes the
|
||||
// pause useful — the frozen view then visibly ages into Stale, so a paused dashboard can
|
||||
// never be mistaken for a live one.
|
||||
if (_paused)
|
||||
return;
|
||||
|
||||
InvokeAsync(() =>
|
||||
{
|
||||
Adopt(snapshot);
|
||||
StateHasChanged();
|
||||
});
|
||||
}
|
||||
|
||||
private void Adopt(OverviewSnapshot snapshot)
|
||||
{
|
||||
_snapshot = snapshot;
|
||||
_instances = snapshot.AllInstances.ToList();
|
||||
}
|
||||
|
||||
private void TogglePause()
|
||||
{
|
||||
_paused = !_paused;
|
||||
|
||||
if (!_paused)
|
||||
Adopt(Store.Current);
|
||||
}
|
||||
|
||||
private int Count(DateTimeOffset now, InstanceStatus status) =>
|
||||
_instances.Count(i => i.Status == status && !i.IsStale(now));
|
||||
|
||||
private int StaleCount(DateTimeOffset now) => _instances.Count(i => i.IsStale(now));
|
||||
|
||||
private string Meta
|
||||
{
|
||||
get
|
||||
{
|
||||
var generated = _snapshot.GeneratedUtc == default
|
||||
? "never"
|
||||
: _snapshot.GeneratedUtc.ToLocalTime().ToString("HH:mm:ss", System.Globalization.CultureInfo.InvariantCulture);
|
||||
|
||||
return $"last sweep {generated} · {_instances.Count} instances";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Groups an application's instances for display: named cluster groups first in registry
|
||||
/// order, then the ungrouped ones.
|
||||
/// </summary>
|
||||
/// <param name="application">The application to lay out.</param>
|
||||
/// <returns>The instance groups.</returns>
|
||||
private static IEnumerable<IGrouping<string?, InstanceSnapshot>> Layout(ApplicationSnapshot application) =>
|
||||
application.Instances
|
||||
.GroupBy(i => i.Group, StringComparer.Ordinal)
|
||||
.OrderBy(g => g.Key is null)
|
||||
.ThenBy(g => g.Key, StringComparer.Ordinal);
|
||||
|
||||
private static string AppMeta(ApplicationSnapshot application)
|
||||
{
|
||||
var clusters = application.Groups.Count;
|
||||
var instances = application.Instances.Count;
|
||||
var clusterText = clusters == 0 ? "" : $" · {clusters} cluster{(clusters == 1 ? "" : "s")}";
|
||||
|
||||
return $"{instances} instance{(instances == 1 ? "" : "s")}{clusterText}";
|
||||
}
|
||||
|
||||
private static string GroupMeta(IGrouping<string?, InstanceSnapshot> group, DateTimeOffset now)
|
||||
{
|
||||
var members = group.Count();
|
||||
var missing = group.Count(i => i.Status is InstanceStatus.Unreachable || i.IsStale(now));
|
||||
|
||||
return $"{members} member{(members == 1 ? "" : "s")} · {missing} unreachable";
|
||||
}
|
||||
|
||||
private static bool IsLeader(ApplicationSnapshot application, InstanceSnapshot instance) =>
|
||||
instance.Group is { } group &&
|
||||
application.GroupFor(group)?.LeaderInstance is { } leader &&
|
||||
string.Equals(leader, instance.Instance, StringComparison.Ordinal);
|
||||
|
||||
private static StatusState RollupState(ApplicationSnapshot application) => application.Worst switch
|
||||
{
|
||||
InstanceStatus.Up => StatusState.Ok,
|
||||
InstanceStatus.Degraded => StatusState.Warn,
|
||||
InstanceStatus.Down or InstanceStatus.Unreachable => StatusState.Bad,
|
||||
_ => StatusState.Idle,
|
||||
};
|
||||
|
||||
private static string RollupLabel(ApplicationSnapshot application) => application.Worst switch
|
||||
{
|
||||
InstanceStatus.Up => "Up",
|
||||
InstanceStatus.Degraded => "Degraded",
|
||||
InstanceStatus.Down => "Down",
|
||||
InstanceStatus.Unreachable => "Unreachable",
|
||||
_ => "Pending",
|
||||
};
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Dispose()
|
||||
{
|
||||
// Both are required: an un-detached handler would call StateHasChanged on a disposed
|
||||
// renderer once per sweep for the life of the process.
|
||||
Store.Changed -= OnSnapshotChanged;
|
||||
_timer?.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
@* Plain RouteView, not AuthorizeRouteView. The dashboard is anonymous and
|
||||
read-only by requirement (design §2): there is no authentication pipeline to
|
||||
authorize against, and wiring one in would be the opposite of the decision. *@
|
||||
<Router AppAssembly="@typeof(Routes).Assembly">
|
||||
<Found Context="routeData">
|
||||
<RouteView RouteData="@routeData" DefaultLayout="@typeof(MainLayout)" />
|
||||
<FocusOnNavigate RouteData="@routeData" Selector="h1" />
|
||||
</Found>
|
||||
<NotFound>
|
||||
<LayoutView Layout="@typeof(MainLayout)">
|
||||
<PageTitle>Overview — Not Found</PageTitle>
|
||||
<section class="app-section">
|
||||
<h1 class="page-head">Not Found</h1>
|
||||
<p class="mb-0">The requested page does not exist.</p>
|
||||
</section>
|
||||
</LayoutView>
|
||||
</NotFound>
|
||||
</Router>
|
||||
@@ -0,0 +1,431 @@
|
||||
@* One instance = one card. Every value shown here comes from the snapshot the
|
||||
poller published; the component never probes anything itself. *@
|
||||
|
||||
<article class="inst @StateClass" data-testid="instance-card" data-instance="@Instance.Instance">
|
||||
<div class="inst-head">
|
||||
<div>
|
||||
<span class="inst-name">@Instance.Instance</span>
|
||||
<span class="inst-host">@HostAndPort</span>
|
||||
</div>
|
||||
<div class="inst-chips">
|
||||
@if (IsLeader)
|
||||
{
|
||||
<StatusPill State="StatusState.Info">Leader</StatusPill>
|
||||
}
|
||||
@if (Instance.Active is ActiveState.Active)
|
||||
{
|
||||
<StatusPill State="StatusState.Info">Active</StatusPill>
|
||||
}
|
||||
else if (Instance.Active is ActiveState.Standby)
|
||||
{
|
||||
<StatusPill State="StatusState.Idle">Standby</StatusPill>
|
||||
}
|
||||
else if (Instance.Active is ActiveState.Unknown)
|
||||
{
|
||||
<StatusPill State="StatusState.Idle">Role ?</StatusPill>
|
||||
}
|
||||
<StatusPill State="StatusPillState">@StatusLabel</StatusPill>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="inst-body">
|
||||
<div class="kv">
|
||||
<span class="k">Latency</span>
|
||||
<span class="v">@LatencyText</span>
|
||||
</div>
|
||||
<div class="kv">
|
||||
<span class="k">@(IsStale ? "Stale since" : "Last poll")</span>
|
||||
<span class="v @(IsStale ? "idle" : null)">@LastPollText</span>
|
||||
</div>
|
||||
<div class="kv">
|
||||
<span class="k">Checks</span>
|
||||
<span class="v @ChecksClass">@ChecksText</span>
|
||||
</div>
|
||||
|
||||
@if (SessionText is { Length: > 0 } sessionText)
|
||||
{
|
||||
@* Only the gateway publishes these counts, so the row is absent everywhere else
|
||||
rather than showing an empty or zero value on apps the concept does not apply to. *@
|
||||
<div class="kv" data-testid="mxaccess-sessions">
|
||||
<span class="k">MXAccess</span>
|
||||
<span class="v @SessionClass">@sessionText</span>
|
||||
</div>
|
||||
}
|
||||
|
||||
@if (HistorianPoolText is { Length: > 0 } poolText)
|
||||
{
|
||||
@* "pooled" and not "connections": this is warm pool depth, and an idle pool sitting at
|
||||
0 is normal. Labelling it as connections would read as an outage. *@
|
||||
<div class="kv" data-testid="historian-pools">
|
||||
<span class="k">Sessions pooled</span>
|
||||
<span class="v @HistorianPoolClass" title="Warm pre-authenticated historian sessions per pool">@poolText</span>
|
||||
</div>
|
||||
}
|
||||
|
||||
@if (Instance.Error is { Length: > 0 } error)
|
||||
{
|
||||
<div class="kv">
|
||||
<span class="k">Error</span>
|
||||
<span class="v bad" title="@error">@Truncate(error, 48)</span>
|
||||
</div>
|
||||
}
|
||||
|
||||
@if (Entries.Count > 0)
|
||||
{
|
||||
<details class="checks">
|
||||
<summary>@Entries.Count check@(Entries.Count == 1 ? "" : "s")</summary>
|
||||
@foreach (var entry in Entries)
|
||||
{
|
||||
<div class="check-row">
|
||||
<span class="check-dot @DotClass(entry.Value.Status)"></span>
|
||||
<span class="check-name">@entry.Key</span>
|
||||
<span class="check-desc">@entry.Value.Description</span>
|
||||
</div>
|
||||
}
|
||||
@if (ClusterDataLine is { Length: > 0 } clusterData)
|
||||
{
|
||||
@* The cluster view, rendered verbatim rather than interpreted: when a pair
|
||||
disagrees about the leader this line is the raw evidence an operator needs
|
||||
to see, not a summary of it. *@
|
||||
<div class="check-data">@((MarkupString)clusterData)</div>
|
||||
}
|
||||
</details>
|
||||
}
|
||||
</div>
|
||||
|
||||
<div class="panel-foot">
|
||||
@if (Instance.ManagementUrl is { Length: > 0 } managementUrl)
|
||||
{
|
||||
<a href="@managementUrl" target="_blank" rel="noopener">Open @Instance.ManagementLabel ↗</a>
|
||||
}
|
||||
else
|
||||
{
|
||||
<span class="foot-note">no management link</span>
|
||||
}
|
||||
<span class="foot-note" data-testid="raw-signals">@RawSignals</span>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
@code {
|
||||
/// <summary>The instance to render.</summary>
|
||||
[Parameter, EditorRequired] public InstanceSnapshot Instance { get; set; } = default!;
|
||||
|
||||
/// <summary>The render-time clock, passed down so every card on a page agrees on "now".</summary>
|
||||
[Parameter, EditorRequired] public DateTimeOffset Now { get; set; }
|
||||
|
||||
/// <summary>Whether this instance is its group's leader.</summary>
|
||||
[Parameter] public bool IsLeader { get; set; }
|
||||
|
||||
private bool IsStale => Instance.IsStale(Now);
|
||||
|
||||
private IReadOnlyList<KeyValuePair<string, ZbHealthEntry>> Entries =>
|
||||
Instance.Report?.Entries.OrderBy(e => e.Key, StringComparer.Ordinal).ToList() ?? [];
|
||||
|
||||
/// <summary>
|
||||
/// Stale wins over the stored status in the class, so the card visibly greys out. What it
|
||||
/// shows is still the last real reading — the point is to mark it as no longer current, not
|
||||
/// to replace it with a guess.
|
||||
/// </summary>
|
||||
private string StateClass => Instance.Status switch
|
||||
{
|
||||
null => "pending",
|
||||
_ when IsStale => $"{Instance.Status.ToString()!.ToLowerInvariant()} stale",
|
||||
var status => status.ToString()!.ToLowerInvariant(),
|
||||
};
|
||||
|
||||
private string StatusLabel => Instance.Status switch
|
||||
{
|
||||
null => "Pending",
|
||||
_ when IsStale => "Stale",
|
||||
InstanceStatus.Up => "Up",
|
||||
InstanceStatus.Degraded => "Degraded",
|
||||
InstanceStatus.Down => "Down",
|
||||
_ => "Unreachable",
|
||||
};
|
||||
|
||||
private StatusState StatusPillState => Instance.Status switch
|
||||
{
|
||||
null => StatusState.Idle,
|
||||
_ when IsStale => StatusState.Idle,
|
||||
InstanceStatus.Up => StatusState.Ok,
|
||||
InstanceStatus.Degraded => StatusState.Warn,
|
||||
_ => StatusState.Bad,
|
||||
};
|
||||
|
||||
private string HostAndPort
|
||||
{
|
||||
get
|
||||
{
|
||||
if (!Uri.TryCreate(Instance.BaseUrl, UriKind.Absolute, out var uri))
|
||||
return Instance.BaseUrl;
|
||||
|
||||
return uri.IsDefaultPort ? uri.Host : $"{uri.Host}:{uri.Port}";
|
||||
}
|
||||
}
|
||||
|
||||
private string LatencyText =>
|
||||
Instance.Status is null ? "—" : $"{Instance.Latency.TotalMilliseconds:0} ms";
|
||||
|
||||
private string LastPollText
|
||||
{
|
||||
get
|
||||
{
|
||||
// Stale cards count from the last successful contact, not the last poll attempt:
|
||||
// "stale since 4 s ago" while an instance has actually been gone for ten minutes
|
||||
// would be actively misleading.
|
||||
var reference = IsStale ? Instance.LastReachableUtc ?? Instance.LastPolledUtc : Instance.LastPolledUtc;
|
||||
return reference is { } at ? Relative(Now - at) : "never";
|
||||
}
|
||||
}
|
||||
|
||||
private string ChecksText
|
||||
{
|
||||
get
|
||||
{
|
||||
if (Entries.Count == 0)
|
||||
return Instance.Status is null ? "—" : "no detail";
|
||||
|
||||
var healthy = Entries.Count(e => string.Equals(e.Value.Status, "Healthy", StringComparison.Ordinal));
|
||||
return $"{healthy} / {Entries.Count} healthy";
|
||||
}
|
||||
}
|
||||
|
||||
private string? ChecksClass
|
||||
{
|
||||
get
|
||||
{
|
||||
if (Entries.Count == 0)
|
||||
return null;
|
||||
|
||||
if (Entries.Any(e => string.Equals(e.Value.Status, "Unhealthy", StringComparison.Ordinal)))
|
||||
return "bad";
|
||||
|
||||
return Entries.Any(e => string.Equals(e.Value.Status, "Degraded", StringComparison.Ordinal))
|
||||
? "warn"
|
||||
: "ok";
|
||||
}
|
||||
}
|
||||
|
||||
private string RawSignals
|
||||
{
|
||||
get
|
||||
{
|
||||
var ready = Instance.Status switch
|
||||
{
|
||||
null => "—",
|
||||
InstanceStatus.Unreachable => "fail",
|
||||
InstanceStatus.Down => "503",
|
||||
_ => "200",
|
||||
};
|
||||
|
||||
var active = Instance.Active switch
|
||||
{
|
||||
ActiveState.Active => "200",
|
||||
ActiveState.Standby => "503",
|
||||
ActiveState.Unknown => "—",
|
||||
_ => null,
|
||||
};
|
||||
|
||||
return active is null ? $"ready {ready}" : $"ready {ready} · active {active}";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Renders the cluster check's data as the mockup's monospace evidence line.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Located by the data it carries rather than by check name: the two apps register the
|
||||
/// same shared checks under different names (<c>akka-cluster</c> vs <c>akka</c>,
|
||||
/// <c>active-node</c> vs <c>cluster-primary</c>), so keying on either one silently drops
|
||||
/// the evidence line for the other.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <c>active</c> and <c>leader</c> are shown side by side <b>on purpose</b>. They come
|
||||
/// from different checks and are different nodes whenever any node has restarted — the
|
||||
/// leader is the lowest-addressed Up member, the active node is the oldest. Showing both
|
||||
/// turns the confusion that produced lmxopcua#494 into something an operator can read
|
||||
/// straight off the card, instead of a silent disagreement between the chip and the tier.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private string? ClusterDataLine
|
||||
{
|
||||
get
|
||||
{
|
||||
var entries = Instance.Report?.Entries.Values;
|
||||
if (entries is null)
|
||||
return null;
|
||||
|
||||
var cluster = entries.FirstOrDefault(e => e.DataString(LeaderResolver.LeaderDataKey) is { Length: > 0 });
|
||||
var active = entries.FirstOrDefault(e => e.DataString(LeaderResolver.ActiveNodeDataKey) is { Length: > 0 });
|
||||
|
||||
if (cluster is null && active is null)
|
||||
return null;
|
||||
|
||||
var parts = new List<string>();
|
||||
|
||||
void Add(string label, string? value)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(value))
|
||||
parts.Add($"<b>{Escape(label)}</b> {Escape(value)}");
|
||||
}
|
||||
|
||||
Add("active", active?.DataString(LeaderResolver.ActiveNodeDataKey));
|
||||
Add("leader", cluster?.DataString(LeaderResolver.LeaderDataKey));
|
||||
Add("members", cluster?.DataInt("memberCount")?.ToString());
|
||||
Add("unreachable", cluster?.DataInt("unreachableCount")?.ToString());
|
||||
Add("self", (cluster ?? active)?.DataString("selfAddress"));
|
||||
|
||||
return parts.Count == 0 ? null : string.Join(" · ", parts);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The MXAccess session check's entry from the <b>active</b> tier, or null when this instance
|
||||
/// publishes none.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Located by the data it carries rather than by check name, for the same reason
|
||||
/// <see cref="ClusterDataLine"/> is: a name is a rename away from silently dropping the
|
||||
/// line, whereas the <c>total</c>/<c>ready</c>/<c>faulted</c> triple is the check's
|
||||
/// actual contract. All three are required so a future check that happens to publish a
|
||||
/// lone <c>total</c> cannot be mistaken for this one.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// It comes from <see cref="InstanceSnapshot.ActiveReport"/>, never the ready tier —
|
||||
/// the gateway registers this check on <c>active</c> because a gateway with no sessions
|
||||
/// is still ready to serve.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private ZbHealthEntry? SessionEntry =>
|
||||
Instance.ActiveReport?.Entries.Values.FirstOrDefault(e =>
|
||||
e.DataInt("total") is not null
|
||||
&& e.DataInt("ready") is not null
|
||||
&& e.DataInt("faulted") is not null);
|
||||
|
||||
/// <summary>The session counts as a short face-level line, or null when there are none to show.</summary>
|
||||
/// <remarks>
|
||||
/// Rendered on the card face rather than inside the collapsed check list: "how many MXAccess
|
||||
/// connections are up" is a number an operator scans for, and a value behind a disclosure
|
||||
/// triangle is one nobody sees. Zero is shown, not hidden — a gateway holding no sessions at
|
||||
/// all is precisely the interesting case, and suppressing it would make the row appear only
|
||||
/// when the news is good.
|
||||
/// </remarks>
|
||||
private string? SessionText
|
||||
{
|
||||
get
|
||||
{
|
||||
var entry = SessionEntry;
|
||||
if (entry is null)
|
||||
return null;
|
||||
|
||||
int total = entry.DataInt("total") ?? 0;
|
||||
if (total == 0)
|
||||
return "0";
|
||||
|
||||
var parts = new List<string> { total.ToString(System.Globalization.CultureInfo.InvariantCulture) };
|
||||
parts.Add($"{entry.DataInt("ready") ?? 0} ready");
|
||||
|
||||
foreach (var (key, label) in new[] { ("faulted", "faulted"), ("starting", "starting") })
|
||||
{
|
||||
if (entry.DataInt(key) is > 0 and var n)
|
||||
parts.Add($"{n} {label}");
|
||||
}
|
||||
|
||||
return string.Join(" · ", parts);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Highlights the session row only when a session is actually broken.</summary>
|
||||
private string? SessionClass => SessionEntry?.DataInt("faulted") is > 0 ? "bad" : null;
|
||||
|
||||
/// <summary>Pools listed in the order an operator cares about; anything unrecognised follows.</summary>
|
||||
private static readonly string[] PoolOrder = ["read", "event"];
|
||||
|
||||
/// <summary>
|
||||
/// The historian session-pool checks, which publish a <c>warmSessions</c> depth per pool.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// These live on the <b>ready</b> tier, not the active one, and carry an entirely
|
||||
/// different key set from the gateway's session check — so the two rows are separate
|
||||
/// renderings rather than one abstraction. Forcing them together would mean inventing a
|
||||
/// shared shape that neither app actually publishes.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>This is pool depth, not connections in use.</b> A pool reporting 0 is idle, not
|
||||
/// broken: sessions are leased on demand and the event pool legitimately sits empty until
|
||||
/// something reads events. The row is labelled "pooled" so nobody reads a 0 as an outage.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private IEnumerable<(string Pool, int Warm, int Failures)> HistorianPools =>
|
||||
(Instance.Report?.Entries.Values ?? Enumerable.Empty<ZbHealthEntry>())
|
||||
.Where(e => e.DataInt("warmSessions") is not null && e.DataString("pool") is { Length: > 0 })
|
||||
.Select(e => (
|
||||
Pool: e.DataString("pool")!,
|
||||
Warm: e.DataInt("warmSessions") ?? 0,
|
||||
Failures: e.DataInt("consecutiveFactoryFailures") ?? 0))
|
||||
.OrderBy(p => Array.IndexOf(PoolOrder, p.Pool) is var i && i >= 0 ? i : int.MaxValue)
|
||||
.ThenBy(p => p.Pool, StringComparer.Ordinal);
|
||||
|
||||
/// <summary>The pool depths as a short face-level line, or null when this app publishes none.</summary>
|
||||
private string? HistorianPoolText
|
||||
{
|
||||
get
|
||||
{
|
||||
var pools = HistorianPools.ToList();
|
||||
if (pools.Count == 0)
|
||||
return null;
|
||||
|
||||
return string.Join(" · ", pools.Select(p =>
|
||||
p.Failures > 0
|
||||
? $"{p.Pool} {p.Warm} ({p.Failures} fail)"
|
||||
: $"{p.Pool} {p.Warm}"));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Red only when a pool's session factory is actually failing — never merely because a pool is
|
||||
/// empty, which is the normal idle state.
|
||||
/// </summary>
|
||||
private string? HistorianPoolClass => HistorianPools.Any(p => p.Failures > 0) ? "bad" : null;
|
||||
|
||||
private static string DotClass(string? status) => status switch
|
||||
{
|
||||
"Healthy" => "ok",
|
||||
"Degraded" => "warn",
|
||||
"Unhealthy" => "bad",
|
||||
_ => "idle",
|
||||
};
|
||||
|
||||
/// <summary>Formats an age the way an operator reads a dashboard: coarse and instant.</summary>
|
||||
/// <param name="age">The elapsed time.</param>
|
||||
/// <returns>A short relative string.</returns>
|
||||
internal static string Relative(TimeSpan age)
|
||||
{
|
||||
if (age < TimeSpan.Zero)
|
||||
age = TimeSpan.Zero;
|
||||
|
||||
if (age.TotalSeconds < 60)
|
||||
return $"{age.TotalSeconds:0} s ago";
|
||||
|
||||
if (age.TotalMinutes < 60)
|
||||
return $"{age.TotalMinutes:0} min ago";
|
||||
|
||||
return age.TotalHours < 24 ? $"{age.TotalHours:0} h ago" : $"{age.TotalDays:0} d ago";
|
||||
}
|
||||
|
||||
private static string Truncate(string value, int max) =>
|
||||
value.Length <= max ? value : string.Concat(value.AsSpan(0, max - 1), "…");
|
||||
|
||||
/// <summary>
|
||||
/// HTML-escapes a value before it goes into the one MarkupString on this card. Health data is
|
||||
/// remote input from another application — trusted operationally, but never trusted to be
|
||||
/// markup-safe.
|
||||
/// </summary>
|
||||
/// <param name="value">The raw value.</param>
|
||||
/// <returns>The escaped value.</returns>
|
||||
private static string Escape(string value) =>
|
||||
System.Net.WebUtility.HtmlEncode(value);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
@using Microsoft.AspNetCore.Components.Routing
|
||||
@using Microsoft.AspNetCore.Components.Web
|
||||
@using ZB.MOM.WW.Overview.Components
|
||||
@using ZB.MOM.WW.Overview.Components.Layout
|
||||
@using ZB.MOM.WW.Overview.Components.Pages
|
||||
@using ZB.MOM.WW.Overview.Components.Widgets
|
||||
@using ZB.MOM.WW.Overview.Polling
|
||||
@using ZB.MOM.WW.Overview.Registry
|
||||
@using ZB.MOM.WW.Theme
|
||||
@using static Microsoft.AspNetCore.Components.Web.RenderMode
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
using ZB.MOM.WW.Health;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Observability;
|
||||
|
||||
/// <summary>Registers the dashboard's own health checks.</summary>
|
||||
public static class ObservabilityServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>Check name for the registry-loaded probe.</summary>
|
||||
public const string RegistryLoadedCheckName = "registry-loaded";
|
||||
|
||||
/// <summary>Check name for the poll-cycle liveness probe.</summary>
|
||||
public const string PollCycleCheckName = "last-poll-cycle-completed";
|
||||
|
||||
/// <summary>
|
||||
/// Adds the two readiness checks that make the dashboard's own <c>/health/ready</c> mean
|
||||
/// something: that it bound a registry, and that it is still polling it.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <returns>The same collection, for chaining.</returns>
|
||||
/// <remarks>
|
||||
/// Both are tagged <see cref="ZbHealthTags.Ready"/> only. The dashboard is a single stateless
|
||||
/// process with no active/standby concept, so it registers nothing on the active tier — its own
|
||||
/// <c>/health/active</c> is deliberately an empty, always-200 report.
|
||||
/// </remarks>
|
||||
public static IServiceCollection AddOverviewHealthChecks(this IServiceCollection services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
services.AddHealthChecks()
|
||||
.AddTypeActivatedCheck<RegistryLoadedHealthCheck>(
|
||||
RegistryLoadedCheckName, failureStatus: null, tags: [ZbHealthTags.Ready])
|
||||
.AddTypeActivatedCheck<PollCycleHealthCheck>(
|
||||
PollCycleCheckName, failureStatus: null, tags: [ZbHealthTags.Ready]);
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Observability;
|
||||
|
||||
/// <summary>
|
||||
/// The dashboard's own Meter: what it currently believes about every registered instance, and how
|
||||
/// long its sweeps take.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The instance gauge is <em>observable</em> — it is read from the published snapshot at scrape
|
||||
/// time rather than written on every poll. The snapshot is already the single authority for what
|
||||
/// the dashboard believes, so deriving the gauge from it makes a disagreement between the page and
|
||||
/// the metric impossible by construction.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The instrument names follow the family convention in
|
||||
/// <c>components/observability/spec/METRIC-CONVENTIONS.md</c> §2 — lower-case, dot-separated
|
||||
/// <c><app>.<subsystem>.<event></c>, durations in seconds — so the Prometheus
|
||||
/// exporter renders them as <c>overview_instance_status</c> and
|
||||
/// <c>overview_poll_duration_seconds</c>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class OverviewMetrics : IDisposable
|
||||
{
|
||||
/// <summary>The app's single Meter name, per the family one-meter-per-app convention.</summary>
|
||||
public const string MeterName = "ZB.MOM.WW.Overview";
|
||||
|
||||
/// <summary>Per-instance status gauge; the value is the <see cref="InstanceStatus"/> ordinal.</summary>
|
||||
public const string InstanceStatusInstrument = "overview.instance.status";
|
||||
|
||||
/// <summary>Sweep-duration histogram, in seconds.</summary>
|
||||
public const string PollDurationInstrument = "overview.poll.duration";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Histogram<double> _pollDuration;
|
||||
private readonly OverviewSnapshotStore _store;
|
||||
|
||||
/// <summary>Initializes the meter and its instruments.</summary>
|
||||
/// <param name="meterFactory">Factory that scopes the meter's lifetime to the container.</param>
|
||||
/// <param name="store">The snapshot the observable gauge reads at scrape time.</param>
|
||||
public OverviewMetrics(IMeterFactory meterFactory, OverviewSnapshotStore store)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(meterFactory);
|
||||
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_meter = meterFactory.Create(MeterName);
|
||||
|
||||
_meter.CreateObservableGauge(
|
||||
InstanceStatusInstrument,
|
||||
ObserveInstanceStatus,
|
||||
unit: "1",
|
||||
description: "Derived status of each registered instance (0=Up, 1=Degraded, 2=Down, 3=Unreachable).");
|
||||
|
||||
_pollDuration = _meter.CreateHistogram<double>(
|
||||
PollDurationInstrument,
|
||||
unit: "s",
|
||||
description: "Wall-clock duration of one poll sweep across every due instance.");
|
||||
}
|
||||
|
||||
/// <summary>Records the duration of one completed sweep.</summary>
|
||||
/// <param name="duration">Wall-clock time the sweep took.</param>
|
||||
public void RecordSweep(TimeSpan duration) => _pollDuration.Record(duration.TotalSeconds);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Dispose() => _meter.Dispose();
|
||||
|
||||
private IEnumerable<Measurement<int>> ObserveInstanceStatus()
|
||||
{
|
||||
foreach (var instance in _store.Current.AllInstances)
|
||||
{
|
||||
// Instances that have never been probed are skipped rather than reported as anything.
|
||||
// Emitting a value for them would publish a status the dashboard does not yet hold —
|
||||
// and the enum's zero ordinal is Up, so the fabrication would be the reassuring one.
|
||||
if (instance.Status is not { } status)
|
||||
continue;
|
||||
|
||||
yield return new Measurement<int>(
|
||||
(int)status,
|
||||
new KeyValuePair<string, object?>("application", instance.Application),
|
||||
|
||||
// Deliberately NOT named "instance": Prometheus attaches its own `instance` label to
|
||||
// every scraped series, and a colliding metric label is silently renamed to
|
||||
// `exported_instance`, which no dashboard query would be written against.
|
||||
new KeyValuePair<string, object?>("node", instance.Instance),
|
||||
new KeyValuePair<string, object?>("group", instance.Group ?? string.Empty));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Observability;
|
||||
|
||||
/// <summary>
|
||||
/// Fails when the poller has stopped sweeping — the one failure mode that would otherwise leave the
|
||||
/// dashboard looking perfectly healthy while every card it serves quietly went stale.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is what makes the dashboard's own <c>/health/ready</c> honest. Without it the process
|
||||
/// answers 200 for as long as Kestrel is up, which says nothing about whether the data behind the
|
||||
/// page is still being refreshed.
|
||||
/// </remarks>
|
||||
public sealed class PollCycleHealthCheck : IHealthCheck
|
||||
{
|
||||
private readonly PollCycleHeartbeat _heartbeat;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly TimeSpan _grace;
|
||||
|
||||
/// <summary>Initializes a new <see cref="PollCycleHealthCheck"/>.</summary>
|
||||
/// <param name="heartbeat">The poller's completed-sweep heartbeat.</param>
|
||||
/// <param name="options">The registry, used to derive the tolerated gap.</param>
|
||||
/// <param name="timeProvider">Clock.</param>
|
||||
public PollCycleHealthCheck(
|
||||
PollCycleHeartbeat heartbeat,
|
||||
IOptions<OverviewOptions> options,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
_heartbeat = heartbeat ?? throw new ArgumentNullException(nameof(heartbeat));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_grace = GraceFor(options.Value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The longest gap between completed sweeps that is still considered healthy.
|
||||
/// </summary>
|
||||
/// <param name="options">The registry.</param>
|
||||
/// <returns>The tolerated gap.</returns>
|
||||
/// <remarks>
|
||||
/// A sweep completes on every timer tick, and the timer ticks at the fastest configured
|
||||
/// interval — so the expected gap is one interval plus however long the probes took, which is
|
||||
/// bounded by the slowest configured timeout. Twice that is comfortably past "slow" and
|
||||
/// unambiguously "stopped".
|
||||
/// </remarks>
|
||||
internal static TimeSpan GraceFor(OverviewOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var timings = options.Applications
|
||||
.SelectMany(app => app.Instances.Select(instance =>
|
||||
EffectiveTimings.Resolve(options, app, instance)))
|
||||
.ToList();
|
||||
|
||||
// An empty registry cannot poll at all; RegistryLoadedHealthCheck is the check that reports
|
||||
// that, so fall back to the global defaults rather than dividing by an empty set here.
|
||||
var interval = timings.Count > 0
|
||||
? timings.Min(t => t.PollInterval)
|
||||
: TimeSpan.FromSeconds(options.PollIntervalSeconds);
|
||||
var timeout = timings.Count > 0
|
||||
? timings.Max(t => t.Timeout)
|
||||
: TimeSpan.FromSeconds(options.TimeoutSeconds);
|
||||
|
||||
return 2 * (interval + timeout);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var last = _heartbeat.LastCompletedUtc;
|
||||
|
||||
var data = new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
["graceSeconds"] = Math.Round(_grace.TotalSeconds, 3),
|
||||
};
|
||||
|
||||
if (last is null)
|
||||
{
|
||||
// Nothing has swept yet. Within the grace window that is ordinary startup, so report
|
||||
// Degraded (still a 200) rather than failing readiness for the first few seconds of
|
||||
// every deployment. Past it, the poller never started and that IS a failure.
|
||||
var waiting = now - _heartbeat.StartedUtc;
|
||||
data["waitingSeconds"] = Math.Round(waiting.TotalSeconds, 3);
|
||||
|
||||
return Task.FromResult(waiting <= _grace
|
||||
? HealthCheckResult.Degraded("No poll cycle has completed yet; the poller is still starting.", data: data)
|
||||
: HealthCheckResult.Unhealthy(
|
||||
$"No poll cycle has completed in {waiting.TotalSeconds:0.#}s since startup.",
|
||||
exception: null,
|
||||
data));
|
||||
}
|
||||
|
||||
var age = now - last.Value;
|
||||
data["lastCycleUtc"] = last.Value.ToString("O", System.Globalization.CultureInfo.InvariantCulture);
|
||||
data["ageSeconds"] = Math.Round(age.TotalSeconds, 3);
|
||||
|
||||
return Task.FromResult(age <= _grace
|
||||
? HealthCheckResult.Healthy($"Last poll cycle completed {age.TotalSeconds:0.#}s ago.", data)
|
||||
: HealthCheckResult.Unhealthy(
|
||||
$"No poll cycle has completed in {age.TotalSeconds:0.#}s; the displayed data is stale.",
|
||||
exception: null,
|
||||
data));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
namespace ZB.MOM.WW.Overview.Observability;
|
||||
|
||||
/// <summary>
|
||||
/// When the poller last finished a full sweep. Written by the poller, read by
|
||||
/// <see cref="PollCycleHealthCheck"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately not derived from the published snapshot's <c>GeneratedUtc</c>: the poller publishes
|
||||
/// the registry once before it probes anything, so a snapshot exists from the moment the host
|
||||
/// starts. A stuck poller would therefore look like a working one. This records only sweeps that
|
||||
/// actually completed.
|
||||
/// </remarks>
|
||||
public sealed class PollCycleHeartbeat
|
||||
{
|
||||
private long _lastCompletedTicks;
|
||||
|
||||
/// <summary>Initializes the heartbeat and stamps the host start time.</summary>
|
||||
/// <param name="timeProvider">Clock.</param>
|
||||
public PollCycleHeartbeat(TimeProvider timeProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
StartedUtc = timeProvider.GetUtcNow();
|
||||
}
|
||||
|
||||
/// <summary>When this heartbeat was constructed — effectively when the host started.</summary>
|
||||
/// <remarks>
|
||||
/// Lets the health check distinguish "started a moment ago and has not swept yet" from "has
|
||||
/// not swept in minutes", which are the same null reading but opposite conclusions.
|
||||
/// </remarks>
|
||||
public DateTimeOffset StartedUtc { get; }
|
||||
|
||||
/// <summary>When the last full sweep completed, or null if none has.</summary>
|
||||
public DateTimeOffset? LastCompletedUtc
|
||||
{
|
||||
get
|
||||
{
|
||||
var ticks = Interlocked.Read(ref _lastCompletedTicks);
|
||||
return ticks == 0 ? null : new DateTimeOffset(ticks, TimeSpan.Zero);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Records that a sweep completed.</summary>
|
||||
/// <param name="at">When it completed.</param>
|
||||
public void RecordCycleCompleted(DateTimeOffset at) =>
|
||||
Interlocked.Exchange(ref _lastCompletedTicks, at.UtcTicks);
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Observability;
|
||||
|
||||
/// <summary>
|
||||
/// Reports how much registry the dashboard actually bound, and fails if it bound none.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The pre-host <c>ConfigPreflight</c> and the options validator already refuse to start on an
|
||||
/// empty registry, so the unhealthy branch is a backstop rather than an expected state. The check
|
||||
/// earns its place on the reported counts: an operator looking at <c>/health/ready</c> after a
|
||||
/// config change can see how many instances this process is watching without reading its
|
||||
/// configuration files.
|
||||
/// </remarks>
|
||||
public sealed class RegistryLoadedHealthCheck : IHealthCheck
|
||||
{
|
||||
private readonly OverviewOptions _options;
|
||||
|
||||
/// <summary>Initializes a new <see cref="RegistryLoadedHealthCheck"/>.</summary>
|
||||
/// <param name="options">The bound registry.</param>
|
||||
public RegistryLoadedHealthCheck(IOptions<OverviewOptions> options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_options = options.Value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var applications = _options.Applications.Count;
|
||||
var instances = _options.Applications.Sum(a => a.Instances.Count);
|
||||
|
||||
var data = new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
["applications"] = applications,
|
||||
["instances"] = instances,
|
||||
};
|
||||
|
||||
return Task.FromResult(instances > 0
|
||||
? HealthCheckResult.Healthy(
|
||||
$"Watching {instances} instance(s) across {applications} application(s).",
|
||||
data)
|
||||
: HealthCheckResult.Unhealthy(
|
||||
"The registry contains no instances; the dashboard has nothing to watch.",
|
||||
exception: null,
|
||||
data));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>The derived state of one probed instance.</summary>
|
||||
public enum InstanceStatus
|
||||
{
|
||||
/// <summary>Ready tier answered 200 / <c>Healthy</c>.</summary>
|
||||
Up,
|
||||
|
||||
/// <summary>Ready tier answered 200 / <c>Degraded</c> — serving, but a check is unhappy.</summary>
|
||||
Degraded,
|
||||
|
||||
/// <summary>
|
||||
/// Ready tier answered 503 / <c>Unhealthy</c>. The app is up and talking; a dependency is not.
|
||||
/// </summary>
|
||||
Down,
|
||||
|
||||
/// <summary>
|
||||
/// Nothing answered — timeout, connection refused, DNS failure, or a body that is not the
|
||||
/// canonical health JSON. Kept distinct from <see cref="Down"/> because it usually means
|
||||
/// network / VPN / registry typo rather than a sick application, and the two lead an operator
|
||||
/// to look in completely different places.
|
||||
/// </summary>
|
||||
Unreachable,
|
||||
}
|
||||
|
||||
/// <summary>Whether a probed instance is the active half of its pair.</summary>
|
||||
public enum ActiveState
|
||||
{
|
||||
/// <summary>The instance has no active/standby concept (single-instance gateways).</summary>
|
||||
NotApplicable,
|
||||
|
||||
/// <summary>The active probe did not answer, so activity is unknown.</summary>
|
||||
Unknown,
|
||||
|
||||
/// <summary>Active tier answered 200 — this is the active node.</summary>
|
||||
Active,
|
||||
|
||||
/// <summary>Active tier answered 503 — this is the standby.</summary>
|
||||
Standby,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure status derivation: probe result → <see cref="InstanceStatus"/>, with two-strike flap
|
||||
/// damping applied across polls. Static and dependency-free so every rule is table-testable.
|
||||
/// </summary>
|
||||
public static class StatusDerivation
|
||||
{
|
||||
/// <summary>
|
||||
/// Consecutive failing observations required before a previously-good instance is shown as
|
||||
/// failing. A single blip — one dropped packet, one GC pause past the timeout — is not news;
|
||||
/// two in a row is.
|
||||
/// </summary>
|
||||
public const int FailureStrikes = 2;
|
||||
|
||||
/// <summary>Classifies one ready-tier probe, before any damping.</summary>
|
||||
/// <param name="probe">The probe result.</param>
|
||||
/// <returns>The observed status.</returns>
|
||||
public static InstanceStatus Observe(HealthProbeResult probe)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(probe);
|
||||
|
||||
if (!probe.Reachable || probe.Report is null)
|
||||
return InstanceStatus.Unreachable;
|
||||
|
||||
// The reported status wins over the HTTP code: they agree by construction (Healthy/Degraded
|
||||
// → 200, Unhealthy → 503), and the body is the more specific of the two — 200 alone cannot
|
||||
// tell Up from Degraded.
|
||||
return probe.Report.Status switch
|
||||
{
|
||||
"Healthy" => InstanceStatus.Up,
|
||||
"Degraded" => InstanceStatus.Degraded,
|
||||
"Unhealthy" => InstanceStatus.Down,
|
||||
|
||||
// Parsed as JSON but carrying no status we recognise: the endpoint is answering with
|
||||
// something that is not this contract, which is the Unreachable case, not a sick app.
|
||||
_ => InstanceStatus.Unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Classifies one active-tier probe.</summary>
|
||||
/// <param name="hasActiveRole">Whether this instance participates in an active/standby pair.</param>
|
||||
/// <param name="probe">The active-tier probe result, or null when not probed.</param>
|
||||
/// <returns>The derived active state.</returns>
|
||||
public static ActiveState ObserveActive(bool hasActiveRole, HealthProbeResult? probe)
|
||||
{
|
||||
if (!hasActiveRole)
|
||||
return ActiveState.NotApplicable;
|
||||
|
||||
if (probe is null || !probe.Reachable || probe.StatusCode is null)
|
||||
return ActiveState.Unknown;
|
||||
|
||||
return probe.StatusCode switch
|
||||
{
|
||||
200 => ActiveState.Active,
|
||||
503 => ActiveState.Standby,
|
||||
_ => ActiveState.Unknown,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies two-strike damping: a failing observation only replaces a previously-good state
|
||||
/// after <see cref="FailureStrikes"/> consecutive failures, while any good observation is
|
||||
/// adopted immediately.
|
||||
/// </summary>
|
||||
/// <param name="previous">The last displayed status, or null on the first-ever poll.</param>
|
||||
/// <param name="previousConsecutiveFailures">Consecutive failures recorded before this poll.</param>
|
||||
/// <param name="observed">This poll's observed status.</param>
|
||||
/// <returns>The status to display and the updated consecutive-failure count.</returns>
|
||||
/// <remarks>
|
||||
/// Recovery is deliberately asymmetric — instant. Damping exists to suppress false alarms, and
|
||||
/// making recovery equally slow would just convert them into false alarms that linger.
|
||||
/// </remarks>
|
||||
public static (InstanceStatus Status, int ConsecutiveFailures) Damp(
|
||||
InstanceStatus? previous,
|
||||
int previousConsecutiveFailures,
|
||||
InstanceStatus observed)
|
||||
{
|
||||
var failing = observed is InstanceStatus.Down or InstanceStatus.Unreachable;
|
||||
|
||||
if (!failing)
|
||||
return (observed, 0);
|
||||
|
||||
var failures = previousConsecutiveFailures + 1;
|
||||
|
||||
// With no previous state there is nothing to protect and nothing better to show: adopt the
|
||||
// observation. Claiming Up on a first poll that failed would be a fabrication.
|
||||
if (previous is null || failures >= FailureStrikes)
|
||||
return (observed, failures);
|
||||
|
||||
return (previous.Value, failures);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>
|
||||
/// Derives per-group leader state from whatever health check a member publishes its cluster view
|
||||
/// under.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Pure and static: given the same instance snapshots it always produces the same answer, which is
|
||||
/// what makes the split-brain rule — the one output an operator would act on at 3am — exhaustively
|
||||
/// testable without a cluster.
|
||||
/// </remarks>
|
||||
public static class LeaderResolver
|
||||
{
|
||||
/// <summary>
|
||||
/// The data key holding the address of the node actually in charge — the oldest Up member,
|
||||
/// published by the active tier's <c>ActiveNodeHealthCheck</c> from ZB.MOM.WW.Health 0.3.0.
|
||||
/// </summary>
|
||||
public const string ActiveNodeDataKey = "activeNode";
|
||||
|
||||
/// <summary>
|
||||
/// The data key holding the Akka cluster leader address, published by
|
||||
/// <c>AkkaClusterHealthCheck</c>. Fallback only — see <see cref="LeaderReportedBy"/>.
|
||||
/// </summary>
|
||||
public const string LeaderDataKey = "leader";
|
||||
|
||||
/// <summary>Resolves leader state for every cluster group present in <paramref name="instances"/>.</summary>
|
||||
/// <param name="instances">One application's instances.</param>
|
||||
/// <returns>Group state in first-appearance order; empty when the application has no groups.</returns>
|
||||
public static IReadOnlyList<GroupSnapshot> Resolve(IEnumerable<InstanceSnapshot> instances)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(instances);
|
||||
|
||||
var all = instances as IReadOnlyList<InstanceSnapshot> ?? instances.ToList();
|
||||
var groups = new List<GroupSnapshot>();
|
||||
|
||||
foreach (var name in all
|
||||
.Select(i => i.Group)
|
||||
.Where(g => !string.IsNullOrWhiteSpace(g))
|
||||
.Distinct(StringComparer.Ordinal))
|
||||
{
|
||||
var members = all.Where(i => string.Equals(i.Group, name, StringComparison.Ordinal)).ToList();
|
||||
|
||||
// A group of instances that have no active/standby concept has no leader to report;
|
||||
// rendering a leader chip for one would invent a distinction that does not exist.
|
||||
if (!members.Exists(m => m.HasActiveRole))
|
||||
continue;
|
||||
|
||||
groups.Add(ResolveGroup(name!, members, all));
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
private static GroupSnapshot ResolveGroup(
|
||||
string name,
|
||||
IReadOnlyList<InstanceSnapshot> members,
|
||||
IReadOnlyList<InstanceSnapshot> allInstances)
|
||||
{
|
||||
// Only members that are actually talking get a vote. A Down or Unreachable node's last
|
||||
// known leader is stale by definition, and counting it would manufacture a disagreement
|
||||
// every time a node goes away mid-failover — the exact moment the flag must stay honest.
|
||||
var votes = members
|
||||
.Where(m => m.Status is InstanceStatus.Up or InstanceStatus.Degraded)
|
||||
.Select(LeaderReportedBy)
|
||||
.Where(address => !string.IsNullOrWhiteSpace(address))
|
||||
.Select(address => address!)
|
||||
.ToList();
|
||||
|
||||
if (votes.Count == 0)
|
||||
return new GroupSnapshot(name, null, null, Disagreement: false, []);
|
||||
|
||||
var distinct = votes.Distinct(StringComparer.Ordinal).OrderBy(a => a, StringComparer.Ordinal).ToList();
|
||||
|
||||
// The most-reported address wins the chip, ties broken ordinally so the display is stable
|
||||
// rather than flickering between two equally-supported claims on consecutive sweeps.
|
||||
var winner = votes
|
||||
.GroupBy(a => a, StringComparer.Ordinal)
|
||||
.OrderByDescending(g => g.Count())
|
||||
.ThenBy(g => g.Key, StringComparer.Ordinal)
|
||||
.First()
|
||||
.Key;
|
||||
|
||||
return new GroupSnapshot(
|
||||
name,
|
||||
ResolveToInstanceName(winner, allInstances),
|
||||
winner,
|
||||
Disagreement: distinct.Count > 1,
|
||||
distinct);
|
||||
}
|
||||
|
||||
/// <summary>Finds the address of the in-charge node one instance reports.</summary>
|
||||
/// <param name="instance">The instance whose report to search.</param>
|
||||
/// <returns>The reported address, or null when this instance publishes none.</returns>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b><c>activeNode</c> first, <c>leader</c> only as a fallback.</b> They are different
|
||||
/// nodes and the chip must show the first. The Akka cluster <i>leader</i> is the
|
||||
/// lowest-addressed Up member — address order, no relationship to time — while the node
|
||||
/// that actually holds the singletons and answers 200 on <c>/health/active</c> is the
|
||||
/// <i>oldest</i> Up member. They agree only until some node restarts, after which the
|
||||
/// restarted node rejoins youngest but keeps its address; observed live on the rebuilt
|
||||
/// OtOpcUa rig, where the leader was <c>central-1</c> while <c>central-2</c> was the
|
||||
/// active node. Ranking <c>leader</c> first would put the Active chip on the standby.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The fallback still matters: it is what an instance running Health < 0.3.0, or one
|
||||
/// whose active tier was not probed, publishes. Showing the leader there is better than
|
||||
/// showing nothing, and it is the pre-0.3.0 behaviour unchanged.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Matched on the DATA KEY, not on the check name. The two apps register the same shared
|
||||
/// checks under different names — ScadaBridge calls its cluster check
|
||||
/// <c>akka-cluster</c>, OtOpcUa calls it <c>akka</c>; the active check is
|
||||
/// <c>active-node</c> in one and <c>cluster-primary</c> in the other — and a resolver
|
||||
/// keyed on any of those names silently renders no chip for the other app, which is
|
||||
/// exactly how this was missed until a live rig showed OtOpcUa's groups bare while
|
||||
/// ScadaBridge's were fully populated. The data key is the part of the contract the
|
||||
/// shared checks actually own.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static string? LeaderReportedBy(InstanceSnapshot instance)
|
||||
{
|
||||
if (instance.Report?.Entries is not { } entries)
|
||||
return null;
|
||||
|
||||
string? leaderFallback = null;
|
||||
|
||||
foreach (var entry in entries.Values)
|
||||
{
|
||||
if (entry.DataString(ActiveNodeDataKey) is { Length: > 0 } activeNode)
|
||||
return activeNode;
|
||||
|
||||
leaderFallback ??= entry.DataString(LeaderDataKey) is { Length: > 0 } leader ? leader : null;
|
||||
}
|
||||
|
||||
return leaderFallback;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps an Akka address to a registry instance name by host.
|
||||
/// </summary>
|
||||
/// <param name="akkaAddress">An address of the form <c>akka.tcp://system@host:port</c>.</param>
|
||||
/// <param name="instances">Candidate instances.</param>
|
||||
/// <returns>The matching instance's name, or null when nothing matches.</returns>
|
||||
/// <remarks>
|
||||
/// Host only, deliberately: the Akka remoting port and the HTTP port are different by design, so
|
||||
/// comparing ports would never match. An unmatched address is not an error — the card falls back
|
||||
/// to showing the raw address, which is still the useful thing to see.
|
||||
/// </remarks>
|
||||
internal static string? ResolveToInstanceName(string akkaAddress, IReadOnlyList<InstanceSnapshot> instances)
|
||||
{
|
||||
var host = HostOf(akkaAddress);
|
||||
if (host is null)
|
||||
return null;
|
||||
|
||||
foreach (var instance in instances)
|
||||
{
|
||||
if (Uri.TryCreate(instance.BaseUrl, UriKind.Absolute, out var uri) &&
|
||||
string.Equals(uri.Host, host, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return instance.Instance;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>Extracts the host from an Akka address.</summary>
|
||||
/// <param name="akkaAddress">The address to parse.</param>
|
||||
/// <returns>The host, or null when the address is not in the expected form.</returns>
|
||||
internal static string? HostOf(string? akkaAddress)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(akkaAddress))
|
||||
return null;
|
||||
|
||||
var at = akkaAddress.LastIndexOf('@');
|
||||
if (at < 0 || at == akkaAddress.Length - 1)
|
||||
return null;
|
||||
|
||||
var hostAndPort = akkaAddress[(at + 1)..];
|
||||
var colon = hostAndPort.LastIndexOf(':');
|
||||
var host = colon > 0 ? hostAndPort[..colon] : hostAndPort;
|
||||
|
||||
return host.Length == 0 ? null : host;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Overview.Observability;
|
||||
using ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>
|
||||
/// The single writer: sweeps every registered instance on a timer and publishes each result set to
|
||||
/// <see cref="OverviewSnapshotStore"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// No retries. A failed probe is simply reported as failed and the next tick is the retry, which
|
||||
/// keeps the sweep bounded in time and keeps the two-strike damping in
|
||||
/// <see cref="StatusDerivation.Damp"/> the only place that decides whether a failure is worth
|
||||
/// showing. Retrying inside a sweep would silently double the effective strike count.
|
||||
/// </remarks>
|
||||
public sealed class OverviewPollerService : BackgroundService
|
||||
{
|
||||
/// <summary>Name of the pooled <see cref="HttpClient"/> used for probes.</summary>
|
||||
public const string HttpClientName = "overview-health";
|
||||
|
||||
/// <summary>
|
||||
/// Slack allowed when deciding whether an instance is due. A timer tick can arrive a hair early
|
||||
/// relative to the recorded sweep time, and without the tolerance such an instance would be
|
||||
/// skipped and end up polling at half its configured rate.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan DueTolerance = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly OverviewSnapshotStore _store;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<OverviewPollerService> _logger;
|
||||
private readonly OverviewMetrics _metrics;
|
||||
private readonly PollCycleHeartbeat _heartbeat;
|
||||
private readonly List<PollTarget> _targets;
|
||||
|
||||
/// <summary>Initializes a new <see cref="OverviewPollerService"/>.</summary>
|
||||
/// <param name="options">The validated registry.</param>
|
||||
/// <param name="httpClientFactory">Factory for the pooled probe client.</param>
|
||||
/// <param name="store">The snapshot store this service writes to.</param>
|
||||
/// <param name="timeProvider">Clock and timer source.</param>
|
||||
/// <param name="metrics">The app's own instruments.</param>
|
||||
/// <param name="heartbeat">Records that a sweep completed, for the self health check.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
public OverviewPollerService(
|
||||
IOptions<OverviewOptions> options,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
OverviewSnapshotStore store,
|
||||
TimeProvider timeProvider,
|
||||
OverviewMetrics metrics,
|
||||
PollCycleHeartbeat heartbeat,
|
||||
ILogger<OverviewPollerService> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_heartbeat = heartbeat ?? throw new ArgumentNullException(nameof(heartbeat));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_targets = BuildTargets(options.Value);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (_targets.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("Overview registry contains no instances; the poller has nothing to do.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Tick at the fastest configured cadence and let each target decide whether it is due.
|
||||
// A single timer honours per-instance overrides without spawning a timer per instance.
|
||||
var tick = _targets.Min(t => t.Timings.PollInterval);
|
||||
|
||||
// Publish the registry before probing anything, so the page paints its full layout at once
|
||||
// instead of appearing empty for the duration of the first sweep.
|
||||
_store.Publish(BuildSnapshot(_timeProvider.GetUtcNow()));
|
||||
|
||||
using var timer = new PeriodicTimer(tick, _timeProvider);
|
||||
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
await SweepAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// Ordinary shutdown.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Runs one sweep: probes every due instance and publishes the resulting snapshot.</summary>
|
||||
/// <param name="cancellationToken">Cancelled on host shutdown.</param>
|
||||
/// <returns>The snapshot that was published.</returns>
|
||||
internal async Task<OverviewSnapshot> SweepAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Checked here rather than left to the transport: whether an in-flight request observes the
|
||||
// token is a property of the handler stack, but "a sweep that started after shutdown must
|
||||
// not publish" has to hold unconditionally, or the store keeps a half-probed registry as
|
||||
// its final state.
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var startedAt = _timeProvider.GetTimestamp();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var due = _targets.Where(t => IsDue(t, now)).ToList();
|
||||
|
||||
if (due.Count > 0)
|
||||
{
|
||||
// One client per sweep, not per probe: IHttpClientFactory pools the handler either way,
|
||||
// and a single instance keeps connection reuse across the fan-out.
|
||||
var client = _httpClientFactory.CreateClient(HttpClientName);
|
||||
var probe = new ZbHealthReportClient(client);
|
||||
|
||||
var results = await Task
|
||||
.WhenAll(due.Select(target => ProbeAsync(probe, target, now, cancellationToken)))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
for (var i = 0; i < due.Count; i++)
|
||||
due[i].Last = results[i];
|
||||
}
|
||||
|
||||
// Re-checked after the fan-out: a shutdown that lands mid-sweep leaves the probed targets
|
||||
// in an arbitrary mix of fresh and stale results, and publishing that mix would freeze a
|
||||
// misleading picture into the store for as long as the process takes to exit.
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var snapshot = BuildSnapshot(now);
|
||||
_store.Publish(snapshot);
|
||||
|
||||
_metrics.RecordSweep(_timeProvider.GetElapsedTime(startedAt));
|
||||
|
||||
// Stamped after the publish, so the heartbeat can never claim a cycle the store did not
|
||||
// receive — the health check's whole job is to notice exactly that gap.
|
||||
_heartbeat.RecordCycleCompleted(_timeProvider.GetUtcNow());
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
private bool IsDue(PollTarget target, DateTimeOffset now) =>
|
||||
target.Last?.LastPolledUtc is not { } polled || now - polled + DueTolerance >= target.Timings.PollInterval;
|
||||
|
||||
private static async Task<InstanceSnapshot> ProbeAsync(
|
||||
ZbHealthReportClient probe,
|
||||
PollTarget target,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var timeout = target.Timings.Timeout;
|
||||
var ready = await probe.ProbeAsync(target.ReadyUrl, timeout, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Skip the active probe when nothing answered the ready probe: it would spend a second full
|
||||
// timeout to learn what we already know, and ObserveActive maps a missing probe to Unknown
|
||||
// anyway. A 503 from ready is NOT a skip — that instance is talking, and whether it is the
|
||||
// active half is exactly what an operator wants to know about a sick node.
|
||||
//
|
||||
// Deliberately NOT gated on HasActiveRole. The active tier answers two different questions
|
||||
// and they have to be separated: (1) am I the active half of a pair, and (2) what does this
|
||||
// tier report about me. A standalone gateway has no answer to (1) but can have plenty to say
|
||||
// about (2) — MxAccessGateway publishes its live MXAccess session counts there. Gating the
|
||||
// probe on the role made that data unreachable, and the tempting fix (declare the gateway
|
||||
// HasActiveRole) is wrong: it would paint an Active/Standby badge asserting a role that does
|
||||
// not exist. So probe unconditionally and let ObserveActive stay role-gated below.
|
||||
HealthProbeResult? active = null;
|
||||
if (ready.Reachable)
|
||||
active = await probe.ProbeStatusAsync(target.ActiveUrl, timeout, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var observed = StatusDerivation.Observe(ready);
|
||||
var (status, failures) = StatusDerivation.Damp(
|
||||
target.Last?.Status,
|
||||
target.Last?.ConsecutiveFailures ?? 0,
|
||||
observed);
|
||||
|
||||
return target.Template with
|
||||
{
|
||||
Status = status,
|
||||
Active = StatusDerivation.ObserveActive(target.Instance.HasActiveRole, active),
|
||||
ConsecutiveFailures = failures,
|
||||
LastPolledUtc = now,
|
||||
LastReachableUtc = ready.Reachable ? now : target.Last?.LastReachableUtc,
|
||||
Latency = ready.Latency,
|
||||
Error = ready.Error,
|
||||
Report = ready.Report,
|
||||
ActiveReport = active?.Report,
|
||||
};
|
||||
}
|
||||
|
||||
private OverviewSnapshot BuildSnapshot(DateTimeOffset now)
|
||||
{
|
||||
var applications = _targets
|
||||
.GroupBy(t => t.Application, StringComparer.Ordinal)
|
||||
.Select(group =>
|
||||
{
|
||||
var instances = group.Select(t => t.Last ?? t.Template).ToList();
|
||||
return new ApplicationSnapshot(group.Key, instances, LeaderResolver.Resolve(instances));
|
||||
})
|
||||
.ToList();
|
||||
|
||||
return new OverviewSnapshot(now, applications);
|
||||
}
|
||||
|
||||
private static List<PollTarget> BuildTargets(OverviewOptions options)
|
||||
{
|
||||
var targets = new List<PollTarget>();
|
||||
|
||||
foreach (var application in options.Applications)
|
||||
{
|
||||
foreach (var instance in application.Instances)
|
||||
{
|
||||
var baseUrl = instance.BaseUrl.TrimEnd('/');
|
||||
|
||||
targets.Add(new PollTarget
|
||||
{
|
||||
Application = application.Name,
|
||||
Instance = instance,
|
||||
Timings = EffectiveTimings.Resolve(options, application, instance),
|
||||
ReadyUrl = $"{baseUrl}/health/ready",
|
||||
ActiveUrl = $"{baseUrl}/health/active",
|
||||
Template = new InstanceSnapshot
|
||||
{
|
||||
Application = application.Name,
|
||||
Instance = instance.Name,
|
||||
Group = instance.Group,
|
||||
BaseUrl = baseUrl,
|
||||
ManagementUrl = instance.ManagementUrl,
|
||||
ManagementLabel = application.ManagementLabel,
|
||||
HasActiveRole = instance.HasActiveRole,
|
||||
StaleAfter = EffectiveTimings.Resolve(options, application, instance).StaleAfter,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return targets;
|
||||
}
|
||||
|
||||
/// <summary>One instance's fixed probe configuration plus its most recent result.</summary>
|
||||
private sealed class PollTarget
|
||||
{
|
||||
public required string Application { get; init; }
|
||||
|
||||
public required InstanceEntry Instance { get; init; }
|
||||
|
||||
public required EffectiveTimings Timings { get; init; }
|
||||
|
||||
public required string ReadyUrl { get; init; }
|
||||
|
||||
public required string ActiveUrl { get; init; }
|
||||
|
||||
/// <summary>The registry-derived fields, reused as the base of every result.</summary>
|
||||
public required InstanceSnapshot Template { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The last published result, or null before the first probe. Written only by the sweep
|
||||
/// loop after its <c>Task.WhenAll</c> has completed, so it is never touched concurrently.
|
||||
/// </summary>
|
||||
public InstanceSnapshot? Last { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>
|
||||
/// Everything the dashboard knows about one instance, as of the last sweep that included it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Immutable by construction: the poller builds a whole new object graph each sweep and swaps it
|
||||
/// into <see cref="OverviewSnapshotStore"/> atomically, so a render can never observe a half-updated
|
||||
/// registry.
|
||||
/// </remarks>
|
||||
public sealed record InstanceSnapshot
|
||||
{
|
||||
/// <summary>Owning application's display name.</summary>
|
||||
public required string Application { get; init; }
|
||||
|
||||
/// <summary>This instance's display name, unique within its application.</summary>
|
||||
public required string Instance { get; init; }
|
||||
|
||||
/// <summary>Cluster group, or null for a standalone instance.</summary>
|
||||
public string? Group { get; init; }
|
||||
|
||||
/// <summary>The probed base URL, shown on the card for diagnosis.</summary>
|
||||
public required string BaseUrl { get; init; }
|
||||
|
||||
/// <summary>Management UI link, when the registry supplies one.</summary>
|
||||
public string? ManagementUrl { get; init; }
|
||||
|
||||
/// <summary>Label for <see cref="ManagementUrl"/>, from the application entry.</summary>
|
||||
public string ManagementLabel { get; init; } = "UI";
|
||||
|
||||
/// <summary>Whether this instance is half of an active/standby pair.</summary>
|
||||
public bool HasActiveRole { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The damped, displayable status — null until the first sweep has actually probed this
|
||||
/// instance.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Null is a real, distinct state and not a stand-in for "Up": the registry is published to the
|
||||
/// store before the first probe returns so the page renders its full layout immediately, and
|
||||
/// showing an unmeasured instance as healthy for that half-second would be a fabricated reading.
|
||||
/// </remarks>
|
||||
public InstanceStatus? Status { get; init; }
|
||||
|
||||
/// <summary>Active/standby state from the active tier.</summary>
|
||||
public ActiveState Active { get; init; } = ActiveState.NotApplicable;
|
||||
|
||||
/// <summary>Consecutive failing observations, carried across sweeps to drive flap damping.</summary>
|
||||
public int ConsecutiveFailures { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When the sweep that produced this snapshot started; null before the first probe. Staleness
|
||||
/// is measured from here, so a stuck or paused poller ages every card even though no probe
|
||||
/// failed.
|
||||
/// </summary>
|
||||
public DateTimeOffset? LastPolledUtc { get; init; }
|
||||
|
||||
/// <summary>When this instance last answered a probe at all, regardless of what it said.</summary>
|
||||
public DateTimeOffset? LastReachableUtc { get; init; }
|
||||
|
||||
/// <summary>Wall-clock duration of the last ready probe.</summary>
|
||||
public TimeSpan Latency { get; init; }
|
||||
|
||||
/// <summary>Why the last probe failed, when it did.</summary>
|
||||
public string? Error { get; init; }
|
||||
|
||||
/// <summary>The last parsed ready-tier body, which drives the per-check detail list.</summary>
|
||||
public ZbHealthReport? Report { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The last parsed <b>active</b>-tier body, when that tier answered with one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Kept separate from <see cref="Report"/> rather than merged into it: the two tiers answer
|
||||
/// different questions, and a check name is only unique within a tier. It is null whenever the
|
||||
/// active tier was not probed, did not answer, or answered without a canonical body — every one
|
||||
/// of which is normal, so nothing driven by it may be required for the card to render.
|
||||
/// </remarks>
|
||||
public ZbHealthReport? ActiveReport { get; init; }
|
||||
|
||||
/// <summary>This instance's effective staleness window.</summary>
|
||||
public TimeSpan StaleAfter { get; init; }
|
||||
|
||||
/// <summary>Whether this snapshot is too old to be trusted as current.</summary>
|
||||
/// <param name="now">The current time.</param>
|
||||
/// <returns>True when the snapshot has aged past its staleness window.</returns>
|
||||
/// <remarks>
|
||||
/// Evaluated at render rather than stored, because staleness is a function of the clock and not
|
||||
/// of anything the poller observed — a snapshot that was fresh when written goes stale on its
|
||||
/// own while nothing at all happens.
|
||||
/// </remarks>
|
||||
public bool IsStale(DateTimeOffset now) =>
|
||||
Status is not null && LastPolledUtc is { } polled && now - polled > StaleAfter;
|
||||
}
|
||||
|
||||
/// <summary>Leader state for one cluster group, derived from what its members report.</summary>
|
||||
/// <param name="Name">The group name from the registry.</param>
|
||||
/// <param name="LeaderInstance">
|
||||
/// Registry name of the instance the reported leader address resolves to, or null when no member
|
||||
/// reported a leader or the address matches no registered instance.
|
||||
/// </param>
|
||||
/// <param name="LeaderAddress">The reported Akka leader address, shown raw when unresolved.</param>
|
||||
/// <param name="Disagreement">
|
||||
/// True when reachable members reported different leaders — the split-brain tell.
|
||||
/// </param>
|
||||
/// <param name="ReportedLeaders">Every distinct address reported, for the disagreement detail.</param>
|
||||
public sealed record GroupSnapshot(
|
||||
string Name,
|
||||
string? LeaderInstance,
|
||||
string? LeaderAddress,
|
||||
bool Disagreement,
|
||||
IReadOnlyList<string> ReportedLeaders);
|
||||
|
||||
/// <summary>One application section of the dashboard.</summary>
|
||||
/// <param name="Name">The application's display name.</param>
|
||||
/// <param name="Instances">Its instances, in registry order.</param>
|
||||
/// <param name="Groups">Leader state per cluster group, in registry order.</param>
|
||||
public sealed record ApplicationSnapshot(
|
||||
string Name,
|
||||
IReadOnlyList<InstanceSnapshot> Instances,
|
||||
IReadOnlyList<GroupSnapshot> Groups)
|
||||
{
|
||||
/// <summary>Finds the group state for <paramref name="group"/>.</summary>
|
||||
/// <param name="group">The group name, or null for standalone instances.</param>
|
||||
/// <returns>The group snapshot, or null when there is none.</returns>
|
||||
public GroupSnapshot? GroupFor(string? group) =>
|
||||
group is null ? null : Groups.FirstOrDefault(g => string.Equals(g.Name, group, StringComparison.Ordinal));
|
||||
|
||||
/// <summary>
|
||||
/// The most severe status across this application's probed instances, for the rollup pill;
|
||||
/// null while nothing has been probed yet.
|
||||
/// </summary>
|
||||
public InstanceStatus? Worst
|
||||
{
|
||||
get
|
||||
{
|
||||
InstanceStatus? worst = null;
|
||||
foreach (var instance in Instances)
|
||||
{
|
||||
if (instance.Status is not { } status)
|
||||
continue;
|
||||
|
||||
if (worst is null || StatusSeverity.Rank(status) > StatusSeverity.Rank(worst.Value))
|
||||
worst = status;
|
||||
}
|
||||
|
||||
return worst;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A complete, atomically-published view of the whole registry.</summary>
|
||||
/// <param name="GeneratedUtc">When the sweep that produced this snapshot started.</param>
|
||||
/// <param name="Applications">The applications, in registry order.</param>
|
||||
public sealed record OverviewSnapshot(DateTimeOffset GeneratedUtc, IReadOnlyList<ApplicationSnapshot> Applications)
|
||||
{
|
||||
/// <summary>The pre-registry snapshot served before the poller has published anything.</summary>
|
||||
public static OverviewSnapshot Empty { get; } = new(default, []);
|
||||
|
||||
/// <summary>Every instance across every application, flattened.</summary>
|
||||
public IEnumerable<InstanceSnapshot> AllInstances => Applications.SelectMany(a => a.Instances);
|
||||
}
|
||||
|
||||
/// <summary>Severity ordering for rollups.</summary>
|
||||
public static class StatusSeverity
|
||||
{
|
||||
/// <summary>Ranks a status; higher is worse.</summary>
|
||||
/// <param name="status">The status to rank.</param>
|
||||
/// <returns>The severity rank.</returns>
|
||||
/// <remarks>
|
||||
/// <see cref="InstanceStatus.Unreachable"/> outranks <see cref="InstanceStatus.Down"/> because
|
||||
/// it carries strictly less information: a Down instance told us what is wrong, an Unreachable
|
||||
/// one did not answer at all and could be anything up to and including gone.
|
||||
/// </remarks>
|
||||
public static int Rank(InstanceStatus status) => status switch
|
||||
{
|
||||
InstanceStatus.Up => 0,
|
||||
InstanceStatus.Degraded => 1,
|
||||
InstanceStatus.Down => 2,
|
||||
InstanceStatus.Unreachable => 3,
|
||||
_ => 3,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>
|
||||
/// Holds the current <see cref="OverviewSnapshot"/> and notifies subscribers when it is replaced.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the cache-first requirement made concrete: the poller is the only writer, and every
|
||||
/// render reads whatever is already here. A page load therefore paints the last known state
|
||||
/// immediately and never waits on — or triggers — a probe of its own.
|
||||
/// </remarks>
|
||||
public sealed class OverviewSnapshotStore
|
||||
{
|
||||
private OverviewSnapshot _current = OverviewSnapshot.Empty;
|
||||
|
||||
/// <summary>Raised after a new snapshot has been published.</summary>
|
||||
/// <remarks>
|
||||
/// Handlers run on the poller's thread, so a Blazor subscriber must marshal onto its own
|
||||
/// synchronization context (<c>InvokeAsync(StateHasChanged)</c>) and must unsubscribe when
|
||||
/// disposed.
|
||||
/// </remarks>
|
||||
public event Action<OverviewSnapshot>? Changed;
|
||||
|
||||
/// <summary>The current snapshot; never null.</summary>
|
||||
public OverviewSnapshot Current => Volatile.Read(ref _current);
|
||||
|
||||
/// <summary>Replaces the current snapshot and notifies subscribers.</summary>
|
||||
/// <param name="snapshot">The new snapshot.</param>
|
||||
public void Publish(OverviewSnapshot snapshot)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(snapshot);
|
||||
|
||||
// A single reference swap of an already-complete graph. Readers hold their own reference
|
||||
// for the duration of a render, so a sweep landing mid-render cannot tear the view.
|
||||
Volatile.Write(ref _current, snapshot);
|
||||
|
||||
Changed?.Invoke(snapshot);
|
||||
}
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using ZB.MOM.WW.Overview.Observability;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>Registers the polling pipeline.</summary>
|
||||
public static class PollingServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>Adds the snapshot store, the probe client and the poller.</summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <returns>The same collection, for chaining.</returns>
|
||||
public static IServiceCollection AddOverviewPolling(this IServiceCollection services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
services.TryAddSingleton(TimeProvider.System);
|
||||
services.TryAddSingleton<OverviewSnapshotStore>();
|
||||
|
||||
// IMeterFactory comes from AddMetrics, which the default web host already registers; the
|
||||
// call is idempotent and makes the dependency explicit for any non-web host.
|
||||
services.AddMetrics();
|
||||
services.TryAddSingleton<OverviewMetrics>();
|
||||
services.TryAddSingleton<PollCycleHeartbeat>();
|
||||
|
||||
services.AddHttpClient(OverviewPollerService.HttpClientName, client =>
|
||||
{
|
||||
// The per-request CancellationTokenSource in ZbHealthReportClient is the only timeout
|
||||
// that should ever fire. Leaving HttpClient's own 100s default in place would let it
|
||||
// pre-empt a per-instance override and report "timed out after 3s" for a 100s wait.
|
||||
client.Timeout = Timeout.InfiniteTimeSpan;
|
||||
|
||||
// Health bodies are small and must not be served from anywhere but the instance itself.
|
||||
client.DefaultRequestHeaders.CacheControl = new() { NoCache = true, NoStore = true };
|
||||
});
|
||||
|
||||
services.AddHostedService<OverviewPollerService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>
|
||||
/// The canonical <c>ZbHealthWriter</c> body every family app serves from <c>MapZbHealth</c>:
|
||||
/// <c>{ status, totalDurationMs, entries: { name: { status, description, durationMs, data? } } }</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This mirrors a contract owned by <c>ZB.MOM.WW.Health</c>. It lives in the app for v1 by
|
||||
/// deliberate decision (design §8.2): promoting it to a <c>ZB.MOM.WW.Health.Client</c> package is
|
||||
/// worth a publish only once a second consumer exists.
|
||||
/// </remarks>
|
||||
public sealed class ZbHealthReport
|
||||
{
|
||||
/// <summary>Aggregate tier status: <c>Healthy</c>, <c>Degraded</c> or <c>Unhealthy</c>.</summary>
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; init; }
|
||||
|
||||
/// <summary>Total time the tier's checks took, in milliseconds.</summary>
|
||||
[JsonPropertyName("totalDurationMs")]
|
||||
public double TotalDurationMs { get; init; }
|
||||
|
||||
/// <summary>Per-check results, keyed by check name (verbatim, e.g. <c>akka-cluster</c>).</summary>
|
||||
[JsonPropertyName("entries")]
|
||||
public Dictionary<string, ZbHealthEntry> Entries { get; init; } = new(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>One check's result inside a <see cref="ZbHealthReport"/>.</summary>
|
||||
public sealed class ZbHealthEntry
|
||||
{
|
||||
/// <summary>This check's status: <c>Healthy</c>, <c>Degraded</c> or <c>Unhealthy</c>.</summary>
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; init; }
|
||||
|
||||
/// <summary>Human-readable detail. Always present in the canonical shape; may be JSON null.</summary>
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
/// <summary>How long this check took, in milliseconds.</summary>
|
||||
[JsonPropertyName("durationMs")]
|
||||
public double DurationMs { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional structured data the check published (ZB.MOM.WW.Health 0.2.0+). Absent entirely on
|
||||
/// checks that publish none and on every pre-0.2.0 node — hence nullable, and hence the
|
||||
/// dashboard must render fully (minus leader) without it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Values stay as <see cref="JsonElement"/>: this is an open contract carrying arbitrary JSON,
|
||||
/// and binding it to concrete types here would break the moment a check adds a field.
|
||||
/// </remarks>
|
||||
[JsonPropertyName("data")]
|
||||
public Dictionary<string, JsonElement>? Data { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Reads <paramref name="key"/> from <see cref="Data"/> as a string, or null when the key is
|
||||
/// absent or is not a JSON string.
|
||||
/// </summary>
|
||||
/// <param name="key">The data key, matched verbatim.</param>
|
||||
/// <returns>The string value, or null.</returns>
|
||||
public string? DataString(string key)
|
||||
{
|
||||
if (Data is null || !Data.TryGetValue(key, out var value))
|
||||
return null;
|
||||
|
||||
return value.ValueKind == JsonValueKind.String ? value.GetString() : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads <paramref name="key"/> from <see cref="Data"/> as an integer, or null when the key is
|
||||
/// absent or is not a JSON number.
|
||||
/// </summary>
|
||||
/// <param name="key">The data key, matched verbatim.</param>
|
||||
/// <returns>The integer value, or null.</returns>
|
||||
public int? DataInt(string key)
|
||||
{
|
||||
if (Data is null || !Data.TryGetValue(key, out var value))
|
||||
return null;
|
||||
|
||||
return value.ValueKind == JsonValueKind.Number && value.TryGetInt32(out var parsed) ? parsed : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>
|
||||
/// One probe of one health endpoint: whether anything answered, what it answered, and how long it
|
||||
/// took.
|
||||
/// </summary>
|
||||
/// <param name="Reachable">
|
||||
/// True when the endpoint answered with a parseable canonical body. False for timeouts, refused
|
||||
/// connections, DNS failures, and bodies that are not the canonical health JSON.
|
||||
/// </param>
|
||||
/// <param name="StatusCode">The HTTP status code, when one was received.</param>
|
||||
/// <param name="Report">The parsed body, when it parsed.</param>
|
||||
/// <param name="Latency">Wall-clock time for the request.</param>
|
||||
/// <param name="Error">A short reason when <paramref name="Reachable"/> is false.</param>
|
||||
public sealed record HealthProbeResult(
|
||||
bool Reachable,
|
||||
int? StatusCode,
|
||||
ZbHealthReport? Report,
|
||||
TimeSpan Latency,
|
||||
string? Error);
|
||||
|
||||
/// <summary>
|
||||
/// GETs a family health endpoint and parses the canonical <c>ZbHealthWriter</c> body.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Both 200 and 503 are treated as SUCCESSFUL probes: an Unhealthy tier still returns a full body,
|
||||
/// and that body is exactly what the dashboard needs to show which check failed. Only a genuine
|
||||
/// non-answer (transport failure) or an unparseable body is a failed probe.
|
||||
/// </remarks>
|
||||
public sealed class ZbHealthReportClient
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
/// <summary>Initializes a new <see cref="ZbHealthReportClient"/>.</summary>
|
||||
/// <param name="httpClient">The HTTP client used for probes.</param>
|
||||
public ZbHealthReportClient(HttpClient httpClient) =>
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
|
||||
/// <summary>Probes <paramref name="url"/> and parses the response body.</summary>
|
||||
/// <param name="url">Absolute URL of a health endpoint.</param>
|
||||
/// <param name="timeout">Per-request timeout.</param>
|
||||
/// <param name="cancellationToken">Token cancelled when the host is shutting down.</param>
|
||||
/// <returns>The probe result; this never throws for an unreachable endpoint.</returns>
|
||||
public Task<HealthProbeResult> ProbeAsync(
|
||||
string url,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
SendAsync(url, timeout, bodyRequired: true, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Probes <paramref name="url"/> for its status code alone, parsing the body only if one
|
||||
/// happens to be there.
|
||||
/// </summary>
|
||||
/// <param name="url">Absolute URL of a health endpoint.</param>
|
||||
/// <param name="timeout">Per-request timeout.</param>
|
||||
/// <param name="cancellationToken">Token cancelled when the host is shutting down.</param>
|
||||
/// <returns>The probe result; this never throws for an unreachable endpoint.</returns>
|
||||
/// <remarks>
|
||||
/// For the active tier the status code IS the answer — 200 means active, 503 means standby —
|
||||
/// and the body carries nothing the dashboard uses. Demanding a parseable body here would
|
||||
/// report every node behind anything that answers the tier with an empty response (a bare
|
||||
/// <c>IActiveNodeGate</c> endpoint, a load balancer, a proxy) as "role unknown" while it is
|
||||
/// plainly telling us the role.
|
||||
/// </remarks>
|
||||
public Task<HealthProbeResult> ProbeStatusAsync(
|
||||
string url,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
SendAsync(url, timeout, bodyRequired: false, cancellationToken);
|
||||
|
||||
private async Task<HealthProbeResult> SendAsync(
|
||||
string url,
|
||||
TimeSpan timeout,
|
||||
bool bodyRequired,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var startedAt = Stopwatch.GetTimestamp();
|
||||
|
||||
// Linked, not a bare timeout token: shutdown must cancel an in-flight probe immediately,
|
||||
// and the two causes have to stay distinguishable so a shutdown is not misreported as a
|
||||
// timed-out instance.
|
||||
using var timeoutSource = new CancellationTokenSource(timeout);
|
||||
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
|
||||
try
|
||||
{
|
||||
using var response = await _httpClient
|
||||
.GetAsync(url, HttpCompletionOption.ResponseContentRead, linked.Token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var body = await response.Content.ReadAsStringAsync(linked.Token).ConfigureAwait(false);
|
||||
var statusCode = (int)response.StatusCode;
|
||||
|
||||
ZbHealthReport? report;
|
||||
try
|
||||
{
|
||||
report = JsonSerializer.Deserialize<ZbHealthReport>(body, SerializerOptions);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
// Answered, but not with this contract — a reverse proxy error page, an HTML login
|
||||
// redirect, a wrong port. That is a registry/plumbing problem, not a sick app, so
|
||||
// it lands as Unreachable rather than Down.
|
||||
return bodyRequired
|
||||
? Failed(startedAt, statusCode, $"response body is not canonical health JSON: {ex.Message}")
|
||||
: new HealthProbeResult(true, statusCode, null, Stopwatch.GetElapsedTime(startedAt), null);
|
||||
}
|
||||
|
||||
if (report is null && bodyRequired)
|
||||
return Failed(startedAt, statusCode, "response body deserialized to null");
|
||||
|
||||
return new HealthProbeResult(true, statusCode, report, Stopwatch.GetElapsedTime(startedAt), null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Host shutdown, not an instance fault — let the caller abandon the sweep.
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return Failed(startedAt, null, $"timed out after {timeout.TotalSeconds:0.##}s");
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
return Failed(startedAt, null, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private static HealthProbeResult Failed(long startedAt, int? statusCode, string error) =>
|
||||
new(false, statusCode, null, Stopwatch.GetElapsedTime(startedAt), error);
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
using ZB.MOM.WW.Health;
|
||||
using ZB.MOM.WW.Overview.Components;
|
||||
using ZB.MOM.WW.Overview.Observability;
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
using ZB.MOM.WW.Overview.Registry;
|
||||
using ZB.MOM.WW.Telemetry;
|
||||
using ZB.MOM.WW.Telemetry.Serilog;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
// The host calls this itself, but ONLY when the environment is literally "Development". Running the
|
||||
// build output under any other name — checking a staging registry locally, say — leaves the Theme
|
||||
// RCL's assets under _content/ unresolvable, and the page renders as a blank white screen with its
|
||||
// markup fully present and every stylesheet 404. Calling it unconditionally is a no-op once
|
||||
// published (the manifest is gone and the assets are real files on disk), so the only thing it
|
||||
// changes is that failure.
|
||||
builder.WebHost.UseStaticWebAssets();
|
||||
|
||||
// Pre-host presence check. The validator below covers shape and consistency, but it only runs once
|
||||
// the options are resolved — a registry section that is missing entirely binds to an empty
|
||||
// OverviewOptions, and the failure a reader can act on is "you configured no registry", not
|
||||
// "Applications must have at least 1 entry". Fail here, before the host is built, with the key name.
|
||||
ConfigPreflight.For(builder.Configuration)
|
||||
.RequireValue($"{OverviewOptions.SectionName}:Applications:0:Name")
|
||||
.RequireValue($"{OverviewOptions.SectionName}:Applications:0:Instances:0:BaseUrl")
|
||||
.ThrowIfInvalid();
|
||||
|
||||
builder.Services.AddValidatedOptions<OverviewOptions, OverviewOptionsValidator>(
|
||||
builder.Configuration, OverviewOptions.SectionName);
|
||||
|
||||
builder.AddZbSerilog(o => o.ServiceName = "overview");
|
||||
|
||||
builder.AddZbTelemetry(o =>
|
||||
{
|
||||
o.ServiceName = "overview";
|
||||
|
||||
// Meters is an ALLOWLIST, not a filter: a Meter absent from this array is never registered with
|
||||
// the MeterProvider, and its instruments vanish from /metrics with no warning anywhere. The
|
||||
// dashboard's own gauge is the only thing here that would go missing silently.
|
||||
o.Meters = [OverviewMetrics.MeterName];
|
||||
});
|
||||
|
||||
builder.Services.AddOverviewPolling();
|
||||
builder.Services.AddOverviewHealthChecks();
|
||||
|
||||
builder.Services.AddRazorComponents()
|
||||
.AddInteractiveServerComponents();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
app.UseStaticFiles();
|
||||
|
||||
// No UseAuthentication/UseAuthorization and no cascading auth state: the dashboard is anonymous and
|
||||
// read-only by requirement (design §2). It renders health data that every one of these apps already
|
||||
// serves unauthenticated, and it has no endpoint that mutates anything.
|
||||
//
|
||||
// UseAntiforgery IS required despite that. AddRazorComponents stamps antiforgery metadata onto every
|
||||
// component endpoint unconditionally, and the endpoint middleware hard-fails a request whose endpoint
|
||||
// carries that metadata with no middleware to honour it — so without this line every page returns
|
||||
// 500. The alternative, MapRazorComponents(...).DisableAntiforgery(), would also boot, but it turns
|
||||
// the first form anyone ever adds into a silent CSRF hole. This app has no forms, so the middleware
|
||||
// is inert; it is here so that stays true by default rather than by vigilance.
|
||||
app.UseAntiforgery();
|
||||
|
||||
app.MapRazorComponents<App>()
|
||||
.AddInteractiveServerRenderMode();
|
||||
|
||||
// The dashboard's own three-tier health surface and Prometheus endpoint. Both are anonymous, like
|
||||
// every /health/* endpoint this app polls — which also means one Overview instance can be pointed
|
||||
// at another and would render it correctly.
|
||||
app.MapZbHealth();
|
||||
app.MapZbMetrics();
|
||||
|
||||
await app.RunAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Exposed so <c>WebApplicationFactory<Program></c> can boot the real host in tests
|
||||
/// (top-level statements otherwise compile to an internal entry-point class).
|
||||
/// </summary>
|
||||
public partial class Program;
|
||||
@@ -0,0 +1,124 @@
|
||||
namespace ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
/// <summary>
|
||||
/// The dashboard's registry: which application instances to probe, and how often. Bound from the
|
||||
/// <c>Overview</c> configuration section and validated by <see cref="OverviewOptionsValidator"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every instance is probed the same way — GET <c>{BaseUrl}/health/ready</c>, plus
|
||||
/// <c>{BaseUrl}/health/active</c> when <see cref="InstanceEntry.HasActiveRole"/>. There is
|
||||
/// deliberately no per-instance probe-mode switch: one probe model, one status derivation.
|
||||
/// </remarks>
|
||||
public sealed class OverviewOptions
|
||||
{
|
||||
/// <summary>The configuration section this binds to.</summary>
|
||||
public const string SectionName = "Overview";
|
||||
|
||||
/// <summary>How often the poller sweeps every instance, in seconds.</summary>
|
||||
public int PollIntervalSeconds { get; set; } = 10;
|
||||
|
||||
/// <summary>Per-request timeout, in seconds.</summary>
|
||||
public int TimeoutSeconds { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// How old a snapshot may get before its instance renders as Stale (out of date / offline)
|
||||
/// regardless of its last known status, in seconds. This is what catches a stuck poller, a
|
||||
/// paused refresh or a slept machine — not just failed probes — so it must exceed the poll
|
||||
/// interval or every instance would read Stale between two healthy polls.
|
||||
/// </summary>
|
||||
public int StaleAfterSeconds { get; set; } = 45;
|
||||
|
||||
/// <summary>The registered applications, each with one or more instances.</summary>
|
||||
public IList<ApplicationEntry> Applications { get; set; } = new List<ApplicationEntry>();
|
||||
}
|
||||
|
||||
/// <summary>One registered application (a section on the dashboard).</summary>
|
||||
public sealed class ApplicationEntry
|
||||
{
|
||||
/// <summary>Display name, unique across the registry.</summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Label for the per-instance management link, e.g. "AdminUI" or "Dashboard".</summary>
|
||||
public string ManagementLabel { get; set; } = "UI";
|
||||
|
||||
/// <summary>Overrides <see cref="OverviewOptions.PollIntervalSeconds"/> for this application.</summary>
|
||||
public int? PollIntervalSeconds { get; set; }
|
||||
|
||||
/// <summary>Overrides <see cref="OverviewOptions.TimeoutSeconds"/> for this application.</summary>
|
||||
public int? TimeoutSeconds { get; set; }
|
||||
|
||||
/// <summary>Overrides <see cref="OverviewOptions.StaleAfterSeconds"/> for this application.</summary>
|
||||
public int? StaleAfterSeconds { get; set; }
|
||||
|
||||
/// <summary>This application's instances; at least one is required.</summary>
|
||||
public IList<InstanceEntry> Instances { get; set; } = new List<InstanceEntry>();
|
||||
}
|
||||
|
||||
/// <summary>One probed instance (a card on the dashboard).</summary>
|
||||
public sealed class InstanceEntry
|
||||
{
|
||||
/// <summary>Display name, unique within its application.</summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The cluster group this instance belongs to (e.g. "central", "site-a"). One Akka cluster is
|
||||
/// one group: leader aggregation and the split-brain check are scoped to it. Null for
|
||||
/// standalone instances.
|
||||
/// </summary>
|
||||
public string? Group { get; set; }
|
||||
|
||||
/// <summary>Absolute base URL whose <c>/health/*</c> endpoints are probed.</summary>
|
||||
public string BaseUrl { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Absolute URL of this instance's own management UI, linked from its card.</summary>
|
||||
public string? ManagementUrl { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this instance participates in an active/standby pair. False for the single-instance
|
||||
/// gateways, which have no such concept — their cards show status without an Active badge and
|
||||
/// are never probed for <c>/health/active</c>.
|
||||
/// </summary>
|
||||
public bool HasActiveRole { get; set; }
|
||||
|
||||
/// <summary>Overrides the application and global poll interval for this instance.</summary>
|
||||
public int? PollIntervalSeconds { get; set; }
|
||||
|
||||
/// <summary>Overrides the application and global request timeout for this instance.</summary>
|
||||
public int? TimeoutSeconds { get; set; }
|
||||
|
||||
/// <summary>Overrides the application and global staleness window for this instance.</summary>
|
||||
public int? StaleAfterSeconds { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The effective (override-resolved) timings for one instance. Instance wins over application,
|
||||
/// application wins over global — resolved once here so no caller re-implements the precedence.
|
||||
/// </summary>
|
||||
/// <param name="PollInterval">Effective poll cadence.</param>
|
||||
/// <param name="Timeout">Effective per-request timeout.</param>
|
||||
/// <param name="StaleAfter">Effective staleness window.</param>
|
||||
public readonly record struct EffectiveTimings(TimeSpan PollInterval, TimeSpan Timeout, TimeSpan StaleAfter)
|
||||
{
|
||||
/// <summary>Resolves the effective timings for <paramref name="instance"/>.</summary>
|
||||
/// <param name="options">The global registry defaults.</param>
|
||||
/// <param name="application">The instance's application.</param>
|
||||
/// <param name="instance">The instance itself.</param>
|
||||
/// <returns>The resolved timings.</returns>
|
||||
public static EffectiveTimings Resolve(
|
||||
OverviewOptions options,
|
||||
ApplicationEntry application,
|
||||
InstanceEntry instance)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(application);
|
||||
ArgumentNullException.ThrowIfNull(instance);
|
||||
|
||||
return new EffectiveTimings(
|
||||
TimeSpan.FromSeconds(
|
||||
instance.PollIntervalSeconds ?? application.PollIntervalSeconds ?? options.PollIntervalSeconds),
|
||||
TimeSpan.FromSeconds(
|
||||
instance.TimeoutSeconds ?? application.TimeoutSeconds ?? options.TimeoutSeconds),
|
||||
TimeSpan.FromSeconds(
|
||||
instance.StaleAfterSeconds ?? application.StaleAfterSeconds ?? options.StaleAfterSeconds));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
/// <summary>
|
||||
/// Validates the <c>Overview</c> registry. Registered through
|
||||
/// <c>AddValidatedOptions<OverviewOptions, OverviewOptionsValidator></c>, so a malformed
|
||||
/// registry fails the host at startup rather than surfacing as a silently-empty dashboard or a
|
||||
/// first-poll exception.
|
||||
/// </summary>
|
||||
public sealed class OverviewOptionsValidator : OptionsValidatorBase<OverviewOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, OverviewOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
RequirePositive(builder, options.PollIntervalSeconds, "Overview:PollIntervalSeconds");
|
||||
RequirePositive(builder, options.TimeoutSeconds, "Overview:TimeoutSeconds");
|
||||
RequirePositive(builder, options.StaleAfterSeconds, "Overview:StaleAfterSeconds");
|
||||
|
||||
builder.MinCount(options.Applications as IReadOnlyCollection<ApplicationEntry>, 1, "Overview:Applications");
|
||||
|
||||
var seenApplications = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (var a = 0; a < options.Applications.Count; a++)
|
||||
{
|
||||
var application = options.Applications[a];
|
||||
var appPath = $"Overview:Applications:{a}";
|
||||
|
||||
builder.Required(application.Name, $"{appPath}:Name");
|
||||
if (!string.IsNullOrWhiteSpace(application.Name) && !seenApplications.Add(application.Name))
|
||||
builder.Add($"{appPath}:Name '{application.Name}' is duplicated; application names must be unique");
|
||||
|
||||
RequirePositiveIfSet(builder, application.PollIntervalSeconds, $"{appPath}:PollIntervalSeconds");
|
||||
RequirePositiveIfSet(builder, application.TimeoutSeconds, $"{appPath}:TimeoutSeconds");
|
||||
RequirePositiveIfSet(builder, application.StaleAfterSeconds, $"{appPath}:StaleAfterSeconds");
|
||||
|
||||
builder.MinCount(
|
||||
application.Instances as IReadOnlyCollection<InstanceEntry>, 1, $"{appPath}:Instances");
|
||||
|
||||
var seenInstances = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (var i = 0; i < application.Instances.Count; i++)
|
||||
{
|
||||
var instance = application.Instances[i];
|
||||
var path = $"{appPath}:Instances:{i}";
|
||||
|
||||
builder.Required(instance.Name, $"{path}:Name");
|
||||
if (!string.IsNullOrWhiteSpace(instance.Name) && !seenInstances.Add(instance.Name))
|
||||
{
|
||||
builder.Add(
|
||||
$"{path}:Name '{instance.Name}' is duplicated within application "
|
||||
+ $"'{application.Name}'; instance names must be unique per application");
|
||||
}
|
||||
|
||||
RequireAbsoluteHttpUrl(builder, instance.BaseUrl, $"{path}:BaseUrl", required: true);
|
||||
RequireAbsoluteHttpUrl(builder, instance.ManagementUrl, $"{path}:ManagementUrl", required: false);
|
||||
|
||||
RequirePositiveIfSet(builder, instance.PollIntervalSeconds, $"{path}:PollIntervalSeconds");
|
||||
RequirePositiveIfSet(builder, instance.TimeoutSeconds, $"{path}:TimeoutSeconds");
|
||||
RequirePositiveIfSet(builder, instance.StaleAfterSeconds, $"{path}:StaleAfterSeconds");
|
||||
|
||||
// Checked on the EFFECTIVE values, not the globals: an override at either level can
|
||||
// invert the relationship on one instance while the global pair stays sane, and an
|
||||
// instance whose stale window is inside its poll interval would render Stale
|
||||
// between two perfectly healthy polls.
|
||||
var timings = EffectiveTimings.Resolve(options, application, instance);
|
||||
if (timings.StaleAfter <= timings.PollInterval)
|
||||
{
|
||||
builder.Add(
|
||||
$"{path}: effective StaleAfterSeconds ({timings.StaleAfter.TotalSeconds:0.##}) must be "
|
||||
+ $"greater than effective PollIntervalSeconds ({timings.PollInterval.TotalSeconds:0.##}) "
|
||||
+ "— otherwise this instance renders Stale between two healthy polls");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void RequirePositive(ValidationBuilder builder, int value, string field) =>
|
||||
builder.RequireThat(value > 0, $"{field} must be greater than 0");
|
||||
|
||||
private static void RequirePositiveIfSet(ValidationBuilder builder, int? value, string field)
|
||||
{
|
||||
if (value is { } set)
|
||||
RequirePositive(builder, set, field);
|
||||
}
|
||||
|
||||
private static void RequireAbsoluteHttpUrl(
|
||||
ValidationBuilder builder, string? value, string field, bool required)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
if (required)
|
||||
builder.Add($"{field} is required");
|
||||
return;
|
||||
}
|
||||
|
||||
// Absolute + http(s) explicitly: a relative URL binds without complaint and only fails at
|
||||
// the first poll, as an opaque HttpClient error against a dashboard that has no base
|
||||
// address of its own.
|
||||
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri)
|
||||
|| (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps))
|
||||
{
|
||||
builder.Add($"{field} must be an absolute http:// or https:// URL (was '{value}')");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<!--
|
||||
The family overview dashboard: a Blazor Server app that polls every registered instance's
|
||||
anonymous /health/ready + /health/active endpoints and renders one cached page.
|
||||
|
||||
Inline pinned versions, deliberately NO central package management: that is the app
|
||||
convention in this family (mxgw and HistorianGateway carry inline pins too, and a
|
||||
CPM-shaped sweep silently reports such repos as empty).
|
||||
|
||||
NO Auth / Audit / Secrets packages. The dashboard is anonymous and read-only by
|
||||
requirement — there is no login, no mutating action to audit, and nothing secret in the
|
||||
registry. That is a requirement, not an omission; do not add them "while at it".
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<RootNamespace>ZB.MOM.WW.Overview</RootNamespace>
|
||||
<UserSecretsId>zb-mom-ww-overview</UserSecretsId>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- Exposes the polling internals (snapshot store, status derivation, leader resolution)
|
||||
to the test project so they are testable without going through the UI. -->
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.Overview.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="ZB.MOM.WW.Theme" Version="0.4.1" />
|
||||
<PackageReference Include="ZB.MOM.WW.Health" Version="0.3.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Telemetry" Version="0.1.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Telemetry.Serilog" Version="0.1.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Configuration" Version="0.1.0" />
|
||||
<PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
|
||||
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1" />
|
||||
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft.AspNetCore": "Warning"
|
||||
}
|
||||
},
|
||||
|
||||
// Kestrel endpoint config, NOT ASPNETCORE_URLS. Setting both puts Kestrel into
|
||||
// explicit-endpoints mode and silently discards one of them — the family's recurring
|
||||
// URLs-override trap.
|
||||
"Kestrel": {
|
||||
"Endpoints": {
|
||||
"Http": {
|
||||
"Url": "http://localhost:5320"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// Development registry for `dotnet run` ON THE MAC HOST. This is deliberately a SUBSET of the
|
||||
// fleet: the dev rigs publish only some node ports to the host, so these are the instances a
|
||||
// host-run dashboard can genuinely reach. Verified against the running containers 2026-07-24.
|
||||
//
|
||||
// ScadaBridge central pair localhost:9001 / :9002 (container :5000 published)
|
||||
// ScadaBridge site nodes NOT published — their health port (8084) is container-internal
|
||||
// OtOpcUa (all six nodes) NOT published — only Traefik's load-balanced :9200, which
|
||||
// round-robins the central pair and so cannot identify a node
|
||||
// HistorianGateway localhost:5220
|
||||
// MxAccessGateway windev 10.100.0.48:5130 (needs the VPN up)
|
||||
//
|
||||
// Run the containerised dashboard (docker/, environment "Docker") to see the whole fleet: it
|
||||
// joins the rig networks and addresses every node by container DNS name.
|
||||
"Overview": {
|
||||
"PollIntervalSeconds": 10,
|
||||
"TimeoutSeconds": 3,
|
||||
"StaleAfterSeconds": 45,
|
||||
"Applications": [
|
||||
{
|
||||
"Name": "ScadaBridge",
|
||||
"ManagementLabel": "CentralUI",
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "central-a",
|
||||
"Group": "central",
|
||||
"BaseUrl": "http://localhost:9001",
|
||||
"ManagementUrl": "http://localhost:9001/",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "central-b",
|
||||
"Group": "central",
|
||||
"BaseUrl": "http://localhost:9002",
|
||||
"ManagementUrl": "http://localhost:9002/",
|
||||
"HasActiveRole": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "HistorianGateway",
|
||||
"ManagementLabel": "Dashboard",
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "histgw",
|
||||
"BaseUrl": "http://localhost:5220",
|
||||
"ManagementUrl": "http://localhost:5220/",
|
||||
"HasActiveRole": false,
|
||||
|
||||
// Its readiness probe reaches the real historian over the VPN; when that is down the
|
||||
// probe hangs rather than refusing, so it needs more room than the 3s default before
|
||||
// it is honestly called unreachable.
|
||||
"TimeoutSeconds": 10
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "MxAccessGateway",
|
||||
"ManagementLabel": "Dashboard",
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "windev",
|
||||
"BaseUrl": "http://10.100.0.48:5130",
|
||||
"ManagementUrl": "http://10.100.0.48:5130/",
|
||||
"HasActiveRole": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
{
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft.AspNetCore": "Warning"
|
||||
}
|
||||
},
|
||||
|
||||
"Kestrel": {
|
||||
"Endpoints": {
|
||||
"Http": {
|
||||
"Url": "http://+:5320"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// Full-fleet registry for the CONTAINERISED dashboard, which joins the rig networks
|
||||
// (otopcua-dev_default, scadabridge-net, zb-historiangw_default) and therefore reaches every
|
||||
// node by container DNS name — including the ones whose ports are never published to the host.
|
||||
//
|
||||
// Ports below were verified by probing the running rigs on 2026-07-24, not read off the compose
|
||||
// files, because the two disagree:
|
||||
// OtOpcUa central-1/2 :9000 (ASPNETCORE_URLS)
|
||||
// OtOpcUa site-* :8080 (NOT 9000 — the site nodes' explicit Kestrel listeners for
|
||||
// LocalDb sync re-bind the primary HTTP port)
|
||||
// ScadaBridge central :5000
|
||||
// ScadaBridge site :8084 (MetricsPort; MapZbHealth lands here from the site-health work)
|
||||
//
|
||||
// ManagementUrl values stay host-relative (localhost:*) on purpose: they are followed by the
|
||||
// OPERATOR'S BROWSER, not by this container, so container DNS names would be dead links.
|
||||
"Overview": {
|
||||
"PollIntervalSeconds": 10,
|
||||
"TimeoutSeconds": 3,
|
||||
"StaleAfterSeconds": 45,
|
||||
"Applications": [
|
||||
{
|
||||
"Name": "ScadaBridge",
|
||||
"ManagementLabel": "CentralUI",
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "central-a",
|
||||
"Group": "central",
|
||||
"BaseUrl": "http://scadabridge-central-a:5000",
|
||||
"ManagementUrl": "http://localhost:9001/",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "central-b",
|
||||
"Group": "central",
|
||||
"BaseUrl": "http://scadabridge-central-b:5000",
|
||||
"ManagementUrl": "http://localhost:9002/",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-a-a",
|
||||
"Group": "site-a",
|
||||
"BaseUrl": "http://scadabridge-site-a-a:8084",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-a-b",
|
||||
"Group": "site-a",
|
||||
"BaseUrl": "http://scadabridge-site-a-b:8084",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-b-a",
|
||||
"Group": "site-b",
|
||||
"BaseUrl": "http://scadabridge-site-b-a:8084",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-b-b",
|
||||
"Group": "site-b",
|
||||
"BaseUrl": "http://scadabridge-site-b-b:8084",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-c-a",
|
||||
"Group": "site-c",
|
||||
"BaseUrl": "http://scadabridge-site-c-a:8084",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-c-b",
|
||||
"Group": "site-c",
|
||||
"BaseUrl": "http://scadabridge-site-c-b:8084",
|
||||
"HasActiveRole": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "OtOpcUa",
|
||||
"ManagementLabel": "AdminUI",
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "central-1",
|
||||
"Group": "MAIN",
|
||||
"BaseUrl": "http://central-1:9000",
|
||||
"ManagementUrl": "http://localhost:9200/",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "central-2",
|
||||
"Group": "MAIN",
|
||||
"BaseUrl": "http://central-2:9000",
|
||||
"ManagementUrl": "http://localhost:9200/",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-a-1",
|
||||
"Group": "SITE-A",
|
||||
"BaseUrl": "http://site-a-1:8080",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-a-2",
|
||||
"Group": "SITE-A",
|
||||
"BaseUrl": "http://site-a-2:8080",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-b-1",
|
||||
"Group": "SITE-B",
|
||||
"BaseUrl": "http://site-b-1:8080",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
"Name": "site-b-2",
|
||||
"Group": "SITE-B",
|
||||
"BaseUrl": "http://site-b-2:8080",
|
||||
"HasActiveRole": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "HistorianGateway",
|
||||
"ManagementLabel": "Dashboard",
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "histgw",
|
||||
"BaseUrl": "http://zb-historiangw:5220",
|
||||
"ManagementUrl": "http://localhost:5220/",
|
||||
"HasActiveRole": false,
|
||||
"TimeoutSeconds": 10
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "MxAccessGateway",
|
||||
"ManagementLabel": "Dashboard",
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "windev",
|
||||
"BaseUrl": "http://10.100.0.48:5130",
|
||||
"ManagementUrl": "http://10.100.0.48:5130/",
|
||||
"HasActiveRole": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
{
|
||||
// Registry for the wonder-app-vd03 deployment (ASPNETCORE_ENVIRONMENT=Wonder), which runs as a
|
||||
// Windows service under NSSM rather than in a container. Named "Wonder" and not "Production" on
|
||||
// purpose, mirroring appsettings.Docker.json: this is ONE deployment's fleet, not a statement
|
||||
// about what any production deployment looks like. A second box gets its own overlay.
|
||||
//
|
||||
// Every port below was established by probing the running box on 2026-08-11 — curl against each
|
||||
// candidate port — NOT read off a compose file or inferred from the docker rig, whose numbers
|
||||
// differ. The rig's ScadaBridge central serves health on :5000; here it is :8085.
|
||||
//
|
||||
// OtOpcUa :9000 service OtOpcUaHost (AdminUI on the same listener)
|
||||
// ScadaBridge Central:8085 service ScadaBridge-Central (CentralUI on the same listener)
|
||||
// ScadaBridge Site :8084 service ScadaBridge-Site (no UI: / returns 404)
|
||||
// MxAccessGateway :5130 service MxAccessGw (dashboard; :5120 is gRPC)
|
||||
// HistorianGateway :5220 service HistorianGateway (dashboard; :5221 is gRPC h2c)
|
||||
//
|
||||
// Ports NOT to probe, confirmed 400 (h2c gRPC, which rejects an HTTP/1.1 health GET):
|
||||
// :8083 ScadaBridge central control plane, :8086.
|
||||
//
|
||||
// ── Why every BaseUrl is 127.0.0.1 and never "localhost" ────────────────────────────────────
|
||||
// LeaderResolver.ResolveToInstanceName maps a reported Akka leader address to an instance by
|
||||
// matching on HOST ONLY — correct by design, since an app's Akka remoting port and its HTTP
|
||||
// port differ, so comparing ports would never match. That resolution assumes what is true on
|
||||
// the docker rig and false here: that instances have DISTINCT hosts. All five of these live on
|
||||
// one box, and the scan is fleet-wide in registry order, so the first entry sharing the host
|
||||
// wins the chip for EVERY group.
|
||||
//
|
||||
// Observed live with BaseUrl "localhost": the ScadaBridge *site* card claimed "Leader · central"
|
||||
// — two separate Akka clusters, so that name was simply wrong.
|
||||
//
|
||||
// There is no host value that makes resolution CORRECT here, because the instances really are
|
||||
// co-located. So the deliberate choice is to make it never match: ScadaBridge's Akka addresses
|
||||
// use the host "localhost" and OtOpcUa's uses the FQDN, and "127.0.0.1" is string-unequal to
|
||||
// both. Every leader chip then falls back to the raw Akka address, which is unambiguous and
|
||||
// always true. A raw address beats a confident wrong name.
|
||||
//
|
||||
// Using the FQDN for OtOpcUa's BaseUrl WOULD resolve its chip correctly, and was rejected: the
|
||||
// name resolves to a link-local IPv6 on this box, and routing the actual health probe through
|
||||
// that to improve a label trades the primary function for a cosmetic one.
|
||||
|
||||
"Serilog": {
|
||||
// Absolute, and deliberately OUTSIDE the Server directory the upgrade procedure replaces.
|
||||
// A relative path resolves against the service's AppDirectory (…\Overview\Server), so every
|
||||
// upgrade would silently discard the log history — the same class of defect that cost
|
||||
// HistorianGateway its secrets store on a swap.
|
||||
"WriteTo": [
|
||||
{ "Name": "Console" },
|
||||
{
|
||||
"Name": "File",
|
||||
"Args": {
|
||||
"path": "E:\\ApiInstall\\Overview\\logs\\overview-.log",
|
||||
"rollingInterval": "Day"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
"Kestrel": {
|
||||
// Bind explicitly; never also set ASPNETCORE_URLS, which would put Kestrel into
|
||||
// explicit-endpoints mode with one of the two silently discarded.
|
||||
"Endpoints": {
|
||||
"Http": {
|
||||
"Url": "http://+:5320"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
"Overview": {
|
||||
"PollIntervalSeconds": 10,
|
||||
"TimeoutSeconds": 3,
|
||||
"StaleAfterSeconds": 45,
|
||||
"Applications": [
|
||||
{
|
||||
"Name": "OtOpcUa",
|
||||
"ManagementLabel": "AdminUI",
|
||||
"Instances": [
|
||||
{
|
||||
// A one-node cluster, so Group is still its real cluster identity: leader aggregation
|
||||
// is scoped to it and the split-brain check cannot false-positive on a single member
|
||||
// (it triggers on more than one DISTINCT reported address).
|
||||
"Name": "wonder",
|
||||
"Group": "MAIN",
|
||||
// Probed by THIS service, on this box — loopback is the shortest path and is immune
|
||||
// to any host firewall rule. ManagementUrl is the opposite case: the operator's
|
||||
// browser follows it from their own machine, so it must carry the box address.
|
||||
"BaseUrl": "http://127.0.0.1:9000",
|
||||
"ManagementUrl": "http://10.220.157.247:9000/",
|
||||
"HasActiveRole": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "ScadaBridge",
|
||||
"ManagementLabel": "CentralUI",
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "central",
|
||||
"Group": "central",
|
||||
"BaseUrl": "http://127.0.0.1:8085",
|
||||
"ManagementUrl": "http://10.220.157.247:8085/",
|
||||
"HasActiveRole": true
|
||||
},
|
||||
{
|
||||
// Central and site are SEPARATE Akka clusters, so they are separate groups; putting
|
||||
// them in one would make the dashboard compare two unrelated leaders and report a
|
||||
// disagreement that does not exist.
|
||||
"Name": "site",
|
||||
"Group": "site",
|
||||
"BaseUrl": "http://127.0.0.1:8084",
|
||||
// No ManagementUrl: the site node serves no UI (/ returns 404), and a card linking to
|
||||
// a 404 is worse than a card with no link.
|
||||
"HasActiveRole": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "MxAccessGateway",
|
||||
"ManagementLabel": "Dashboard",
|
||||
"Instances": [
|
||||
{
|
||||
// HasActiveRole false: mxgw is a standalone gateway with no cluster, so an Active
|
||||
// badge here would assert a role that does not exist.
|
||||
//
|
||||
// This flag no longer decides whether the active tier is PROBED — since 2026-08-12 it
|
||||
// always is, for any reachable instance. That tier is where mxgw publishes its live
|
||||
// MXAccess session counts (the "MXAccess" row on the card), and gating the probe on the
|
||||
// role made those counts unreachable. Do NOT flip this to true to get them back: the
|
||||
// probe and the badge are deliberately separate, and true would paint a false badge.
|
||||
//
|
||||
// (The previous note here said the active tier ran an EMPTY check set. That was true
|
||||
// when written and stopped being true when the gateway added its session check.)
|
||||
"Name": "wonder",
|
||||
"BaseUrl": "http://127.0.0.1:5130",
|
||||
"ManagementUrl": "http://10.220.157.247:5130/",
|
||||
"HasActiveRole": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "HistorianGateway",
|
||||
"ManagementLabel": "Dashboard",
|
||||
"Instances": [
|
||||
{
|
||||
"Name": "wonder",
|
||||
"BaseUrl": "http://127.0.0.1:5220",
|
||||
"ManagementUrl": "http://10.220.157.247:5220/",
|
||||
"HasActiveRole": false,
|
||||
// Its readiness probe reaches the historian and the Galaxy SQL DB over the network,
|
||||
// so it is the one instance that legitimately exceeds the 3s global timeout.
|
||||
"TimeoutSeconds": 10
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user