Merge branch 'fix/r3-04-one-role-authority' — R3-04: EffectiveRoles is the one role authority (Cluster:Roles wins over OTOPCUA_ROLES, normalised, validated, fail-start on unknown/overlay-changed roles); Program.cs + all four validators consume it; case-insensitive gossip predicates; source-scan guard (archreview 01/S-12 = 03/C5, 01/C-9)
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
@@ -36,12 +37,37 @@ namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
/// </remarks>
|
||||
public sealed class AkkaClusterOptionsValidator : OptionsValidatorBase<AkkaClusterOptions>
|
||||
{
|
||||
private readonly EffectiveRoles? _effectiveRoles;
|
||||
private readonly IConfiguration? _configuration;
|
||||
|
||||
/// <summary>
|
||||
/// DI-constructed by <c>AddValidatedOptions</c> (a plain <c>AddSingleton</c>), so constructor
|
||||
/// dependencies are safe here. Both <paramref name="effectiveRoles"/> and
|
||||
/// <paramref name="configuration"/> are optional (default <see langword="null"/>) so a caller
|
||||
/// with neither to hand — including every test in this file that pre-dates R3-04 — keeps
|
||||
/// constructing this validator parameterlessly; the post-overlay guard below (choice 4) is
|
||||
/// simply skipped when either is null.
|
||||
/// </summary>
|
||||
/// <param name="effectiveRoles">The registered role authority, or null to skip the post-overlay guard.</param>
|
||||
/// <param name="configuration">
|
||||
/// The live application configuration — deliberately NOT read through
|
||||
/// <paramref name="effectiveRoles"/>'s own snapshot (review finding, 2026-08-20): the guard
|
||||
/// needs to see a role overlay applied AFTER <paramref name="effectiveRoles"/> was resolved,
|
||||
/// and only a fresh read off the live <see cref="IConfiguration"/> can observe that.
|
||||
/// </param>
|
||||
public AkkaClusterOptionsValidator(EffectiveRoles? effectiveRoles = null, IConfiguration? configuration = null)
|
||||
{
|
||||
_effectiveRoles = effectiveRoles;
|
||||
_configuration = configuration;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, AkkaClusterOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
ValidateBootstrapGuard(builder, options.BootstrapGuard);
|
||||
ValidateAgainstEffectiveRoles(builder, options);
|
||||
|
||||
var seeds = options.SeedNodes ?? Array.Empty<string>();
|
||||
if (seeds.Length == 0)
|
||||
@@ -89,6 +115,85 @@ public sealed class AkkaClusterOptionsValidator : OptionsValidatorBase<AkkaClust
|
||||
builder.Add($"Cluster:BootstrapGuard:ProbeConnectTimeoutMs must be > 0 when the guard is enabled (was {guard.ProbeConnectTimeoutMs}).");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// R3-04 (choice 4, the residual divergence surface): <c>Program.cs</c> resolves
|
||||
/// <see cref="EffectiveRoles"/> BEFORE the per-role appsettings overlay loads (the overlay's
|
||||
/// filename depends on the answer), so if a role overlay — or a <c>${secret:}</c> expansion —
|
||||
/// ever set <c>Cluster:Roles</c> itself, that value would silently disagree with the
|
||||
/// <see cref="EffectiveRoles"/> instance every other consumer holds.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// ⚠️ <b>Review finding (2026-08-20): deliberately NOT <paramref name="options"/>.Roles.</b>
|
||||
/// <see cref="ServiceCollectionExtensions.AddOtOpcUaCluster"/>'s <c>PostConfigure</c>
|
||||
/// unconditionally sets <c>options.Roles = [.. effectiveRoles.Gossiped]</c> — the very
|
||||
/// structural-identity fix T2 landed — so <paramref name="options"/>.Roles and
|
||||
/// <see cref="_effectiveRoles"/>.Gossiped are EQUAL BY CONSTRUCTION on every real host.
|
||||
/// Comparing them here would make this guard structurally unreachable: it could never
|
||||
/// observe a divergence no matter what an overlay did, because the same PostConfigure step
|
||||
/// that a later overlay's Cluster:Roles bind feeds into is exactly what overwrites the
|
||||
/// difference away before this validator ever runs. Caught by
|
||||
/// <c>An_overlay_that_changes_Cluster_Roles_after_resolution_fails_naming_both_views</c>,
|
||||
/// which failed (no throw) against the options.Roles-comparing version of this method.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Instead this re-reads the RAW <c>Cluster:Roles</c> section directly off the live
|
||||
/// <see cref="_configuration"/>, which — being the same mutable object
|
||||
/// <c>Program.cs</c>'s <c>builder.Configuration</c> is — reflects every provider added up
|
||||
/// to the point something first resolves <c>IOptions<AkkaClusterOptions></c>,
|
||||
/// INCLUDING a role overlay applied after <see cref="_effectiveRoles"/> was captured.
|
||||
/// Skipped when that raw value is empty — the ordinary shape for every
|
||||
/// <c>OTOPCUA_ROLES</c>-only node, where nothing meaningful lives under the
|
||||
/// <c>Cluster:Roles</c> key either before OR after an overlay, and comparing an empty
|
||||
/// array against a non-empty <c>Gossiped</c> list would be a permanent false positive for
|
||||
/// every env-only deployment (which CLAUDE.md notes is most of production).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Compares as SETS (case-insensitively) rather than sequence, because ordering carries no
|
||||
/// meaning here and a re-ordering is not the hazard this rule exists to catch — a
|
||||
/// genuinely different role set is. A no-op today for the "nothing changed" case: none of
|
||||
/// the four shipped overlays carries a <c>Cluster</c> section.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private void ValidateAgainstEffectiveRoles(ValidationBuilder builder, AkkaClusterOptions options)
|
||||
{
|
||||
_ = options; // intentionally unused — see remarks: comparing against it would be vacuous.
|
||||
if (_effectiveRoles is null || _configuration is null) return;
|
||||
|
||||
var postOverlayRaw = _configuration.GetSection(EffectiveRoles.ConfiguredRolesKey).Get<string[]>();
|
||||
if (postOverlayRaw is not { Length: > 0 })
|
||||
{
|
||||
// Cluster:Roles is empty post-overlay too -> nothing to compare. Correct for every
|
||||
// OTOPCUA_ROLES-only node, not merely "no data yet".
|
||||
return;
|
||||
}
|
||||
|
||||
string[] normalized;
|
||||
try
|
||||
{
|
||||
// Reuses EffectiveRoles' own configured-path normalization/validation rather than
|
||||
// re-implementing it (archreview theme 4 — one rule, stated once).
|
||||
normalized = [.. EffectiveRoles.Resolve(postOverlayRaw, envRoles: null).Gossiped];
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
builder.Add(
|
||||
"Cluster:Roles was changed by a configuration layer applied after role resolution to "
|
||||
+ $"an invalid value ({ex.Message}). A per-role appsettings overlay may not set Cluster:Roles.");
|
||||
return;
|
||||
}
|
||||
|
||||
var bound = new HashSet<string>(normalized, StringComparer.OrdinalIgnoreCase);
|
||||
var registered = new HashSet<string>(_effectiveRoles.Gossiped, StringComparer.OrdinalIgnoreCase);
|
||||
if (bound.SetEquals(registered)) return;
|
||||
|
||||
builder.Add(
|
||||
"Cluster:Roles was changed by a configuration layer applied after role resolution "
|
||||
+ $"(host view: [{string.Join(",", _effectiveRoles.Gossiped)}], bound: "
|
||||
+ $"[{string.Join(",", normalized)}]). A per-role appsettings "
|
||||
+ "overlay may not set Cluster:Roles.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="seed"/> addresses this node itself — host AND port.
|
||||
/// </summary>
|
||||
|
||||
@@ -23,8 +23,22 @@ public sealed class ClusterRoleInfo : IClusterRoleInfo, IDisposable
|
||||
private readonly string? _clusterRole;
|
||||
private readonly string? _clusterId;
|
||||
private readonly object _lock = new();
|
||||
// R3-04 (01/C-9, census row 17) — the two dictionaries below have DIFFERENT correct comparers,
|
||||
// and that split is deliberate, not an oversight (review finding, 2026-08-20).
|
||||
//
|
||||
// _membersByRole answers "every member carrying (a spelling of) this role" — OrdinalIgnoreCase,
|
||||
// so a PEER running an older build that still gossips a miscased plain role during a rolling
|
||||
// upgrade is not invisible to MembersWithRole("driver"). Merging casings IS the point here.
|
||||
//
|
||||
// _roleLeaders must stay Ordinal. Akka computes — and raises RoleLeaderChanged for — role
|
||||
// leadership PER EXACT GOSSIPED ROLE STRING: "driver" and "Driver" are two entirely distinct
|
||||
// roles to Akka's own role-leader machinery, each independently electing its own leader. An
|
||||
// OrdinalIgnoreCase dictionary here would collapse both onto ONE key, so whichever
|
||||
// RoleLeaderChanged event landed last would silently overwrite the other's leader — turning two
|
||||
// real, independent facts Akka tracks separately into one, wrong, merged one. Pinned by
|
||||
// ClusterRoleInfoRoleLeaderCaseTests.RoleLeaderChanged_for_driver_then_Driver_keeps_distinct_entries.
|
||||
private readonly Dictionary<string, Member?> _roleLeaders = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, HashSet<Member>> _membersByRole = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, HashSet<Member>> _membersByRole = new(StringComparer.OrdinalIgnoreCase);
|
||||
private IActorRef? _subscriber;
|
||||
|
||||
/// <summary>Initializes a new instance of the ClusterRoleInfo class.</summary>
|
||||
|
||||
@@ -15,18 +15,17 @@ namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
/// </remarks>
|
||||
public sealed class ConfigSourceOptionsValidator : IValidateOptions<ConfigSourceOptions>
|
||||
{
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly EffectiveRoles _effectiveRoles;
|
||||
|
||||
/// <summary>
|
||||
/// DI-constructed by <c>AddValidatedOptions</c> (a plain <c>AddSingleton</c>), so a
|
||||
/// constructor dependency is safe here. <see cref="IConfiguration"/> — not
|
||||
/// <c>IClusterRoleInfo</c> — is the source of this node's roles: <c>IClusterRoleInfo</c>'s
|
||||
/// implementation needs the <c>ActorSystem</c>, which does not exist yet at
|
||||
/// <c>ValidateOnStart</c> time.
|
||||
/// constructor dependency is safe here. This validator's only use of configuration is this
|
||||
/// node's roles, so it takes <see cref="EffectiveRoles"/> — the single role authority
|
||||
/// (archreview R3-04) — directly rather than <see cref="IConfiguration"/>.
|
||||
/// </summary>
|
||||
public ConfigSourceOptionsValidator(IConfiguration configuration)
|
||||
public ConfigSourceOptionsValidator(EffectiveRoles effectiveRoles)
|
||||
{
|
||||
_configuration = configuration;
|
||||
_effectiveRoles = effectiveRoles;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -93,15 +92,12 @@ public sealed class ConfigSourceOptionsValidator : IValidateOptions<ConfigSource
|
||||
|
||||
// Per-cluster mesh Phase 4: a driver-only node has no central ConfigDb connection to fall back
|
||||
// on, so Direct is only ever valid on a fused admin+driver node (admin holds the SQL
|
||||
// connection). Roles come from configuration, not IClusterRoleInfo — see the ctor remark — and
|
||||
// through RoleParser.ResolveEffectiveRoles so the OTOPCUA_ROLES fallback applies (Gitea #532).
|
||||
// Reading Cluster:Roles alone exempted the exact node this rule exists for: an env-only
|
||||
// driver-only node has no ConfigDb (Program.cs gates it on the same env var) yet saw an empty
|
||||
// role list here and sailed through on Direct, to read its configuration from nothing.
|
||||
var roles = RoleParser.ResolveEffectiveRoles(
|
||||
_configuration.GetSection("Cluster:Roles").Get<string[]>());
|
||||
var isDriver = Array.IndexOf(roles, RoleParser.Driver) >= 0;
|
||||
var isAdmin = Array.IndexOf(roles, RoleParser.Admin) >= 0;
|
||||
// connection). EffectiveRoles is the single role authority (archreview R3-04): its precedence
|
||||
// already applies the OTOPCUA_ROLES fallback (Gitea #532) and its normalization already folds
|
||||
// case, so neither an env-only driver-only node nor one spelled 'Driver' can sail through here
|
||||
// on Direct to read its configuration from nothing.
|
||||
var isDriver = _effectiveRoles.HasDriver;
|
||||
var isAdmin = _effectiveRoles.HasAdmin;
|
||||
|
||||
if (isDriver && !isAdmin && isDirect)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
/// <summary>Which of the two role sources <see cref="EffectiveRoles"/> resolved from.</summary>
|
||||
public enum RoleSource
|
||||
{
|
||||
/// <summary>Neither source named any role.</summary>
|
||||
None,
|
||||
|
||||
/// <summary><c>Cluster:Roles</c> was non-empty and won outright.</summary>
|
||||
Configured,
|
||||
|
||||
/// <summary><c>Cluster:Roles</c> was empty; <c>OTOPCUA_ROLES</c> supplied the roles.</summary>
|
||||
Environment,
|
||||
}
|
||||
|
||||
/// <summary>The node's effective cluster roles — the SINGLE authority for "what is this node".
|
||||
/// Resolved once, from one documented precedence, and consumed by BOTH the Host's conditional
|
||||
/// wiring and the Akka role list, so the two cannot disagree (archreview 01/S-12 = 03/C5).</summary>
|
||||
/// <remarks>
|
||||
/// <para><b>Precedence:</b> <c>Cluster:Roles</c> when non-empty, else <c>OTOPCUA_ROLES</c>,
|
||||
/// else empty. Unchanged from <see cref="RoleParser.ResolveEffectiveRoles(IReadOnlyList{string}?, string?)"/>
|
||||
/// — what changed is that <c>Program.cs</c> now obeys it too, instead of reading the environment
|
||||
/// directly.</para>
|
||||
/// <para><b>Normalization is two-layer, on purpose.</b> <see cref="Gossiped"/> lower-cases plain
|
||||
/// roles (Akka matches gossiped role names ORDINALLY, and every consumer compares against a
|
||||
/// lowercase constant) but keeps a cluster role's ClusterId suffix VERBATIM — lower-casing that
|
||||
/// would rename <c>cluster-SITE-A</c> on a running fleet, and a rolling pair upgrade would gossip
|
||||
/// two different role names mid-window. Lower-casing a plain role changes the wire only for a
|
||||
/// node that is already broken (archreview 01/S-12 = 03/C5, 01/C-9).</para>
|
||||
/// <para><b>Validation runs on BOTH sources.</b> The environment path was always validated by
|
||||
/// <see cref="RoleParser.Parse"/>; the configured path previously accepted anything verbatim
|
||||
/// (<c>Cluster:Roles: ["Drivr"]</c> silently gossiped a role that matched nothing). Both now fail
|
||||
/// fast, naming the offending key, on an unknown role — a miscased but KNOWN role still
|
||||
/// normalizes silently rather than failing, or an operator's <c>Driver</c> would brick the node
|
||||
/// at the exact moment they upgrade.</para>
|
||||
/// </remarks>
|
||||
public sealed class EffectiveRoles
|
||||
{
|
||||
private EffectiveRoles(IReadOnlyList<string> gossiped, RoleSource source, string? disagreementWarning)
|
||||
{
|
||||
Gossiped = gossiped;
|
||||
Source = source;
|
||||
DisagreementWarning = disagreementWarning;
|
||||
ClusterRole = RoleParser.ClusterRoleFromRoles(gossiped);
|
||||
ClusterId = ClusterRole is null ? null : RoleParser.ClusterIdFromRole(ClusterRole);
|
||||
}
|
||||
|
||||
/// <summary>The configuration key <see cref="Resolve(IConfiguration)"/> reads for the configured
|
||||
/// side of the precedence.</summary>
|
||||
public const string ConfiguredRolesKey = "Cluster:Roles";
|
||||
|
||||
/// <summary>The roles as they are handed to Akka — <see cref="AkkaClusterOptions.Roles"/> becomes
|
||||
/// this list, unmodified.</summary>
|
||||
public IReadOnlyList<string> Gossiped { get; }
|
||||
|
||||
/// <summary>Which source resolved these roles.</summary>
|
||||
public RoleSource Source { get; }
|
||||
|
||||
/// <summary>True when <see cref="RoleParser.Admin"/> is among <see cref="Gossiped"/> (case-insensitive).</summary>
|
||||
public bool HasAdmin => Has(RoleParser.Admin);
|
||||
|
||||
/// <summary>True when <see cref="RoleParser.Driver"/> is among <see cref="Gossiped"/> (case-insensitive).</summary>
|
||||
public bool HasDriver => Has(RoleParser.Driver);
|
||||
|
||||
/// <summary>True when <see cref="RoleParser.Dev"/> is among <see cref="Gossiped"/> (case-insensitive).</summary>
|
||||
public bool HasDev => Has(RoleParser.Dev);
|
||||
|
||||
/// <summary>True when <paramref name="role"/> is among <see cref="Gossiped"/>, compared case-insensitively.</summary>
|
||||
/// <param name="role">The role to test for.</param>
|
||||
public bool Has(string role) =>
|
||||
Gossiped.Any(r => string.Equals(r, role, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
/// <summary>The first cluster-scoped role in <see cref="Gossiped"/>, original spelling preserved,
|
||||
/// or null when this node carries none.</summary>
|
||||
public string? ClusterRole { get; }
|
||||
|
||||
/// <summary>The ClusterId this node belongs to (the suffix of <see cref="ClusterRole"/>), or null
|
||||
/// for a legacy / single-mesh node.</summary>
|
||||
public string? ClusterId { get; }
|
||||
|
||||
/// <summary>Non-null when BOTH <c>Cluster:Roles</c> and <c>OTOPCUA_ROLES</c> were set and their
|
||||
/// normalized role sets differ — the <c>Cluster:Roles</c>-wins case an operator almost certainly
|
||||
/// did not intend (choice 5). Observability only: <see cref="Gossiped"/> still resolves from
|
||||
/// <c>Cluster:Roles</c> exactly as usual. Returned rather than logged from this static factory —
|
||||
/// the caller (<c>Program.cs</c>) logs it once Serilog is configured.</summary>
|
||||
public string? DisagreementWarning { get; }
|
||||
|
||||
/// <summary>The one factory: resolves the effective roles off <paramref name="configuration"/>'s
|
||||
/// <c>Cluster:Roles</c> and the process's <c>OTOPCUA_ROLES</c> environment variable.</summary>
|
||||
/// <param name="configuration">The application configuration.</param>
|
||||
/// <returns>The resolved <see cref="EffectiveRoles"/>.</returns>
|
||||
/// <exception cref="ArgumentException">Either source names an unknown role.</exception>
|
||||
public static EffectiveRoles Resolve(IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
var configured = configuration.GetSection(ConfiguredRolesKey).Get<string[]>();
|
||||
var envRoles = Environment.GetEnvironmentVariable(RoleParser.RolesEnvVar);
|
||||
return Resolve(configured, envRoles);
|
||||
}
|
||||
|
||||
/// <summary>Resolves the effective roles from explicit values — the overload
|
||||
/// <see cref="Resolve(IConfiguration)"/> delegates to.</summary>
|
||||
/// <param name="configured">The bound <c>Cluster:Roles</c> value.</param>
|
||||
/// <param name="envRoles">The raw <c>OTOPCUA_ROLES</c> value.</param>
|
||||
/// <returns>The resolved <see cref="EffectiveRoles"/>.</returns>
|
||||
/// <exception cref="ArgumentException">Either source names an unknown role.</exception>
|
||||
public static EffectiveRoles Resolve(IReadOnlyList<string>? configured, string? envRoles)
|
||||
{
|
||||
var configuredNormalized = configured is { Count: > 0 }
|
||||
? NormalizeAndValidateConfigured(configured)
|
||||
: null;
|
||||
|
||||
// RoleParser.Parse already trims, lower-cases (fully — including a cluster suffix, which is
|
||||
// intentionally different from the configured path; see ClusterIdFromRole's remarks), dedupes
|
||||
// and validates.
|
||||
var envNormalized = !string.IsNullOrWhiteSpace(envRoles)
|
||||
? RoleParser.Parse(envRoles)
|
||||
: null;
|
||||
|
||||
var warning = BuildDisagreementWarning(configuredNormalized, envNormalized);
|
||||
|
||||
if (configuredNormalized is not null)
|
||||
return new EffectiveRoles(configuredNormalized, RoleSource.Configured, warning);
|
||||
|
||||
if (envNormalized is not null)
|
||||
return new EffectiveRoles(envNormalized, RoleSource.Environment, warning);
|
||||
|
||||
return new EffectiveRoles(Array.Empty<string>(), RoleSource.None, null);
|
||||
}
|
||||
|
||||
/// <summary>Choice 5: a Warning (not an error) when both sources are set and their normalized
|
||||
/// role sets differ — this cannot fire when only one source is set, so it is always <see
|
||||
/// langword="null"/> in that case regardless of the other source's absence.</summary>
|
||||
private static string? BuildDisagreementWarning(string[]? configuredNormalized, string[]? envNormalized)
|
||||
{
|
||||
if (configuredNormalized is null || envNormalized is null) return null;
|
||||
|
||||
var configuredSet = new HashSet<string>(configuredNormalized, StringComparer.OrdinalIgnoreCase);
|
||||
var envSet = new HashSet<string>(envNormalized, StringComparer.OrdinalIgnoreCase);
|
||||
if (configuredSet.SetEquals(envSet)) return null;
|
||||
|
||||
return $"{ConfiguredRolesKey} ([{string.Join(",", configuredNormalized)}]) and "
|
||||
+ $"{RoleParser.RolesEnvVar} ([{string.Join(",", envNormalized)}]) disagree; "
|
||||
+ $"{ConfiguredRolesKey} wins outright.";
|
||||
}
|
||||
|
||||
/// <summary>Normalizes and validates the <c>Cluster:Roles</c> entries: a plain role is trimmed and
|
||||
/// lower-cased; a cluster role keeps its ClusterId suffix verbatim but gets a normalized
|
||||
/// (lower-case) <c>cluster-</c> prefix. Throws naming the offending index and value on anything
|
||||
/// that is neither.</summary>
|
||||
private static string[] NormalizeAndValidateConfigured(IReadOnlyList<string> configured)
|
||||
{
|
||||
var result = new string[configured.Count];
|
||||
for (var i = 0; i < configured.Count; i++)
|
||||
{
|
||||
var raw = configured[i];
|
||||
var trimmed = raw?.Trim() ?? string.Empty;
|
||||
|
||||
if (trimmed.Length == 0)
|
||||
{
|
||||
throw Unknown(i, raw);
|
||||
}
|
||||
|
||||
if (RoleParser.IsClusterRole(trimmed))
|
||||
{
|
||||
var clusterId = RoleParser.ClusterIdFromRole(trimmed);
|
||||
if (clusterId.Length == 0)
|
||||
{
|
||||
throw Unknown(i, raw);
|
||||
}
|
||||
|
||||
result[i] = RoleParser.ClusterRolePrefix + clusterId;
|
||||
continue;
|
||||
}
|
||||
|
||||
var lowered = trimmed.ToLowerInvariant();
|
||||
if (lowered is not (RoleParser.Admin or RoleParser.Driver or RoleParser.Dev))
|
||||
{
|
||||
throw Unknown(i, raw);
|
||||
}
|
||||
|
||||
result[i] = lowered;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static ArgumentException Unknown(int index, string? raw) =>
|
||||
new($"{ConfiguredRolesKey}[{index}] = '{raw}' is not a known role. Allowed: {RoleParser.Admin}, "
|
||||
+ $"{RoleParser.Driver}, {RoleParser.Dev}, or '{RoleParser.ClusterRolePrefix}<ClusterId>'.");
|
||||
}
|
||||
@@ -17,19 +17,21 @@ namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
public sealed class MeshTransportOptionsValidator : IValidateOptions<MeshTransportOptions>
|
||||
{
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly EffectiveRoles _effectiveRoles;
|
||||
|
||||
/// <summary>
|
||||
/// DI-constructed by <c>AddValidatedOptions</c> (a plain <c>AddSingleton</c>), so a
|
||||
/// constructor dependency is safe here. <see cref="IConfiguration"/> — not
|
||||
/// <c>IClusterRoleInfo</c> — is the source of this node's roles: <c>IClusterRoleInfo</c>'s
|
||||
/// implementation needs the <c>ActorSystem</c>, which does not exist yet at
|
||||
/// <c>ValidateOnStart</c> time. Same shape as <see cref="ConfigSourceOptionsValidator"/> and
|
||||
/// <see cref="TelemetryOptionsValidator"/>.
|
||||
/// DI-constructed by <c>AddValidatedOptions</c> (a plain <c>AddSingleton</c>), so constructor
|
||||
/// dependencies are safe here. <see cref="IConfiguration"/> is still needed for the
|
||||
/// <c>ConfigServe:GrpcListenPort</c> cross-read (the admin-side obligation below) —
|
||||
/// <see cref="EffectiveRoles"/> is the single role authority (archreview R3-04) this rule reads
|
||||
/// roles through instead of re-deriving them.
|
||||
/// </summary>
|
||||
/// <param name="configuration">The application configuration, read for <c>Cluster:Roles</c>.</param>
|
||||
public MeshTransportOptionsValidator(IConfiguration configuration)
|
||||
/// <param name="configuration">The application configuration, read for <c>ConfigServe:GrpcListenPort</c>.</param>
|
||||
/// <param name="effectiveRoles">This node's resolved effective roles.</param>
|
||||
public MeshTransportOptionsValidator(IConfiguration configuration, EffectiveRoles effectiveRoles)
|
||||
{
|
||||
_configuration = configuration;
|
||||
_effectiveRoles = effectiveRoles;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -68,15 +70,12 @@ public sealed class MeshTransportOptionsValidator : IValidateOptions<MeshTranspo
|
||||
// follows it is a separate scope). Scoped to the driver role because an admin-only central node
|
||||
// dials nodes rather than being dialled, and resolves per-cluster keys at runtime through
|
||||
// MeshTransportOptions.ClusterPsks / the Secrets store, so requiring a single GrpcPsk and an
|
||||
// endpoint list of it would demand configuration it never reads. Roles come from
|
||||
// configuration, not IClusterRoleInfo (see the ctor remark), and through
|
||||
// RoleParser.ResolveEffectiveRoles so the OTOPCUA_ROLES fallback applies (Gitea #532) — a node
|
||||
// configured only through the env var is still the driver that has to dial central, and
|
||||
// exempting it here would leave its acks rejected unauthenticated at runtime.
|
||||
var roles = RoleParser.ResolveEffectiveRoles(
|
||||
_configuration.GetSection("Cluster:Roles").Get<string[]>());
|
||||
var isDriver = Array.IndexOf(roles, RoleParser.Driver) >= 0;
|
||||
var isAdmin = Array.IndexOf(roles, RoleParser.Admin) >= 0;
|
||||
// endpoint list of it would demand configuration it never reads. EffectiveRoles is the single
|
||||
// role authority (archreview R3-04) — its precedence already applies the OTOPCUA_ROLES
|
||||
// fallback and its normalization already folds case, so a node configured only through the env
|
||||
// var, or spelled 'Driver', is still the driver that has to dial central.
|
||||
var isDriver = _effectiveRoles.HasDriver;
|
||||
var isAdmin = _effectiveRoles.HasAdmin;
|
||||
|
||||
if (isGrpc && isDriver)
|
||||
{
|
||||
|
||||
@@ -111,12 +111,15 @@ public static class RoleParser
|
||||
/// <param name="configuredRoles">The bound <c>Cluster:Roles</c> value.</param>
|
||||
/// <param name="envRoles">The raw <c>OTOPCUA_ROLES</c> value.</param>
|
||||
/// <returns>The effective roles; empty when neither source names any.</returns>
|
||||
/// <exception cref="ArgumentException">The environment value names an unknown role.</exception>
|
||||
public static string[] ResolveEffectiveRoles(IReadOnlyList<string>? configuredRoles, string? envRoles)
|
||||
{
|
||||
if (configuredRoles is { Count: > 0 }) return [.. configuredRoles];
|
||||
return Parse(envRoles);
|
||||
}
|
||||
/// <exception cref="ArgumentException">Either source names an unknown role.</exception>
|
||||
/// <remarks>
|
||||
/// Thin delegation to <see cref="EffectiveRoles.Resolve(IReadOnlyList{string}?, string?)"/> —
|
||||
/// the single role authority (archreview R3-04). <see cref="EffectiveRoles"/> is the type to
|
||||
/// reach for directly; this overload is retained so existing callers of the array-returning
|
||||
/// shape keep compiling.
|
||||
/// </remarks>
|
||||
public static string[] ResolveEffectiveRoles(IReadOnlyList<string>? configuredRoles, string? envRoles) =>
|
||||
[.. EffectiveRoles.Resolve(configuredRoles, envRoles).Gossiped];
|
||||
|
||||
/// <summary>Reads the effective roles for the current process, resolving the environment itself.</summary>
|
||||
/// <param name="configuredRoles">The bound <c>Cluster:Roles</c> value.</param>
|
||||
@@ -150,4 +153,15 @@ public static class RoleParser
|
||||
|
||||
return roles;
|
||||
}
|
||||
|
||||
/// <summary>True when a GOSSIPED role list carries <paramref name="role"/>, compared
|
||||
/// case-insensitively. Akka compares role names ordinally and a PEER may be running an older
|
||||
/// build that gossips a verbatim 'Driver' (archreview 01/C-9), so every predicate over
|
||||
/// <c>Member.Roles</c> goes through here rather than <c>List.Contains</c>.</summary>
|
||||
/// <param name="gossipedRoles">The gossiped role list (e.g. <c>Member.Roles</c>).</param>
|
||||
/// <param name="role">The role to test for.</param>
|
||||
/// <returns>True when <paramref name="role"/> is present, case-insensitively.</returns>
|
||||
public static bool GossipedRolesHave(IEnumerable<string>? gossipedRoles, string role) =>
|
||||
gossipedRoles is not null
|
||||
&& gossipedRoles.Any(r => string.Equals(r, role, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ using Akka.Logger.Serilog;
|
||||
using Akka.Remote.Hosting;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
@@ -34,21 +35,40 @@ public static class ServiceCollectionExtensions
|
||||
services.AddValidatedOptions<AkkaClusterOptions, AkkaClusterOptionsValidator>(
|
||||
configuration, AkkaClusterOptions.SectionName);
|
||||
|
||||
// Gitea #532: apply the OTOPCUA_ROLES fallback the Roles property has always DOCUMENTED, in the
|
||||
// one place every consumer reads through. Before this, a node configured only through the env
|
||||
// var — the knob that drives all of Program.cs's role wiring — bound an EMPTY Roles array and so
|
||||
// joined the cluster role-less: invisible to RoleLeader/MembersWithRole, ineligible for every
|
||||
// role-scoped singleton, and un-deployable ("no driver-role nodes are Up") with nothing in its
|
||||
// own logs to say so. Doing it here rather than at each call site means BuildClusterOptions
|
||||
// (Akka's own member roles), ClusterRoleInfo, the ControlPlane singleton scope and
|
||||
// ConfigDbApiKeyStore's ClusterId all see the same answer.
|
||||
// R3-04 (structural identity, choice 3): EffectiveRoles is resolved ONCE and registered as a
|
||||
// singleton, off the SAME IConfiguration this call received — not off a DI-resolved
|
||||
// IConfiguration, which a plain ServiceCollection (as opposed to a WebApplicationBuilder) never
|
||||
// registers. Every consumer (Program.cs's hasAdmin/hasDriver, the four role-scoped validators,
|
||||
// AkkaClusterOptions.Roles below) therefore reads the exact same instance, so there is no second
|
||||
// derivation left to disagree with — the boot-guard OVERALL asked for is this identity, not a
|
||||
// runtime assert (see EffectiveRoles + AkkaClusterOptionsValidator's post-overlay guard).
|
||||
// TryAddSingleton keeps this usable from a harness that registers its own EffectiveRoles first.
|
||||
services.TryAddSingleton(_ => EffectiveRoles.Resolve(configuration));
|
||||
|
||||
// Review finding (2026-08-20): AkkaClusterOptionsValidator's post-overlay guard needs to
|
||||
// re-read Cluster:Roles LIVE, off the current (possibly overlay-mutated) configuration — not
|
||||
// off the pre-overlay EffectiveRoles snapshot above, and not off the bound AkkaClusterOptions
|
||||
// (the PostConfigure two lines below makes that equal to the snapshot BY CONSTRUCTION, so
|
||||
// comparing the guard against it could never observe a divergence). A bare ServiceCollection
|
||||
// (unlike WebApplicationBuilder, which auto-registers IConfiguration) never registers this
|
||||
// otherwise; TryAddSingleton is a no-op on a real host, where it is already registered.
|
||||
services.TryAddSingleton(configuration);
|
||||
|
||||
// Gitea #532 (superseded by R3-04): apply the OTOPCUA_ROLES fallback the Roles property has
|
||||
// always DOCUMENTED, in the one place every consumer reads through. Before this, a node
|
||||
// configured only through the env var — the knob that drives all of Program.cs's role wiring —
|
||||
// bound an EMPTY Roles array and so joined the cluster role-less: invisible to
|
||||
// RoleLeader/MembersWithRole, ineligible for every role-scoped singleton, and un-deployable
|
||||
// ("no driver-role nodes are Up") with nothing in its own logs to say so. Doing it here rather
|
||||
// than at each call site means BuildClusterOptions (Akka's own member roles), ClusterRoleInfo,
|
||||
// the ControlPlane singleton scope and ConfigDbApiKeyStore's ClusterId all see the same answer.
|
||||
//
|
||||
// PostConfigure, not a bind trick: it runs after the section binds and before ValidateOnStart,
|
||||
// so the validators below see the effective list. An unknown role in the env var throws out of
|
||||
// RoleParser.Parse — the same throw Program.cs already takes on the same value, so an invalid
|
||||
// env var cannot reach here on the production path in the first place.
|
||||
services.PostConfigure<AkkaClusterOptions>(
|
||||
options => options.Roles = RoleParser.ResolveEffectiveRoles(options.Roles));
|
||||
// so the validators below see the effective list. Now consumes the registered EffectiveRoles
|
||||
// instance rather than re-deriving from options.Roles + the environment — options.Roles becomes
|
||||
// BY CONSTRUCTION the same list Program.cs's hasAdmin/hasDriver are computed from.
|
||||
services.AddOptions<AkkaClusterOptions>().PostConfigure<EffectiveRoles>(
|
||||
(options, roles) => options.Roles = [.. roles.Gossiped]);
|
||||
|
||||
// Per-cluster mesh Phase 2: which transport carries central→node commands. Validated at
|
||||
// startup for the same reason the cluster options are — a contact point that cannot resolve
|
||||
|
||||
@@ -36,58 +36,40 @@ namespace ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
public sealed class SplitTopologyTransportValidator : IValidateOptions<MeshTransportOptions>
|
||||
{
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly EffectiveRoles _effectiveRoles;
|
||||
|
||||
/// <summary>
|
||||
/// DI-constructed by <c>AddValidatedOptions</c> (a plain <c>AddSingleton</c>), so a
|
||||
/// constructor dependency is safe here. <see cref="IConfiguration"/> — not
|
||||
/// <c>IClusterRoleInfo</c> — is the source of this node's roles and the three transport modes:
|
||||
/// <c>IClusterRoleInfo</c>'s implementation needs the <c>ActorSystem</c>, which does not exist
|
||||
/// yet at <c>ValidateOnStart</c> time. Reading the modes from configuration too (rather than
|
||||
/// from three injected <c>IOptions</c>) mirrors how the sibling validators cross-read
|
||||
/// <c>Cluster:Roles</c>, and avoids a service-resolution ordering dependency at validation
|
||||
/// time.
|
||||
/// DI-constructed by <c>AddValidatedOptions</c> (a plain <c>AddSingleton</c>), so constructor
|
||||
/// dependencies are safe here. <see cref="IConfiguration"/> is still needed for the three
|
||||
/// transport-mode cross-reads (<c>MeshTransport:Mode</c>, <c>Telemetry:Mode</c>,
|
||||
/// <c>TelemetryDial:Mode</c>) — not <c>IClusterRoleInfo</c>, whose implementation needs the
|
||||
/// <c>ActorSystem</c>, which does not exist yet at <c>ValidateOnStart</c> time.
|
||||
/// <see cref="EffectiveRoles"/> is the single role authority (archreview R3-04) — this rule no
|
||||
/// longer re-derives the effective, normalized role list itself.
|
||||
/// </summary>
|
||||
public SplitTopologyTransportValidator(IConfiguration configuration)
|
||||
public SplitTopologyTransportValidator(IConfiguration configuration, EffectiveRoles effectiveRoles)
|
||||
{
|
||||
_configuration = configuration;
|
||||
_effectiveRoles = effectiveRoles;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValidateOptionsResult Validate(string? name, MeshTransportOptions options)
|
||||
{
|
||||
// The options instance is intentionally unused: every value this rule depends on is cross-read
|
||||
// from IConfiguration, the same way ConfigSourceOptionsValidator reads Cluster:Roles.
|
||||
//
|
||||
// Resolve the EFFECTIVE Akka roles exactly as the node does, or this "fail loud" rule fires on
|
||||
// the wrong role view and passes silently on the two shapes it exists to catch:
|
||||
// (a) Roles-source divergence. AkkaClusterOptions.Roles binds Cluster:Roles but falls back to
|
||||
// OTOPCUA_ROLES when that is empty. A node configured with only
|
||||
// OTOPCUA_ROLES=driver,cluster-SITE-A genuinely carries the cluster role in Akka, so it
|
||||
// must be validated too. Until Gitea #532 this validator was the ONLY code that applied
|
||||
// that fallback — it is now RoleParser.ResolveEffectiveRoles, shared with the
|
||||
// AkkaClusterOptions PostConfigure that finally makes the option itself agree.
|
||||
// (b) Case. The Cluster:Roles bind path is verbatim (no lowercasing), whereas RoleParser.Parse
|
||||
// lowercases the OTOPCUA_ROLES path. Normalize BOTH here (trim + ToLowerInvariant) so
|
||||
// 'Driver' / 'Admin' cannot slip past the ordinal driver / admin checks. (IsClusterRole
|
||||
// itself is case-insensitive as of #532; the trim is still this rule's own business.)
|
||||
var configured = _configuration.GetSection("Cluster:Roles").Get<string[]>() ?? Array.Empty<string>();
|
||||
var effective = RoleParser.ResolveEffectiveRoles(configured);
|
||||
|
||||
var normalized = Array.ConvertAll(effective, r => r.Trim().ToLowerInvariant());
|
||||
|
||||
var clusterRoleIndex = Array.FindIndex(normalized, RoleParser.IsClusterRole);
|
||||
if (clusterRoleIndex < 0)
|
||||
// from IConfiguration (the three transport modes) or EffectiveRoles (this node's roles).
|
||||
if (_effectiveRoles.ClusterRole is null)
|
||||
{
|
||||
// No cluster-{ClusterId} role → not the split topology → DPS is legitimate. This is the
|
||||
// exemption that keeps the validator a no-op for every legacy / single-mesh / test node.
|
||||
return ValidateOptionsResult.Success;
|
||||
}
|
||||
|
||||
// Classification is normalized (above); the ClusterId in the message is taken from the
|
||||
// operator's original spelling so it names what they actually wrote.
|
||||
var clusterId = RoleParser.ClusterIdFromRole(effective[clusterRoleIndex].Trim());
|
||||
var isDriver = Array.IndexOf(normalized, RoleParser.Driver) >= 0;
|
||||
var isAdmin = Array.IndexOf(normalized, RoleParser.Admin) >= 0;
|
||||
// ClusterId keeps the operator's original spelling (EffectiveRoles preserves it verbatim by
|
||||
// design — see EffectiveRoles' remarks) so the message names what they actually wrote.
|
||||
var clusterId = _effectiveRoles.ClusterId;
|
||||
var isDriver = _effectiveRoles.HasDriver;
|
||||
var isAdmin = _effectiveRoles.HasAdmin;
|
||||
|
||||
var errors = new List<string>();
|
||||
|
||||
|
||||
@@ -87,18 +87,17 @@ public sealed class TelemetryDialOptions
|
||||
/// </remarks>
|
||||
public sealed class TelemetryOptionsValidator : IValidateOptions<TelemetryOptions>
|
||||
{
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly EffectiveRoles _effectiveRoles;
|
||||
|
||||
/// <summary>
|
||||
/// DI-constructed by <c>AddValidatedOptions</c> (a plain <c>AddSingleton</c>), so a
|
||||
/// constructor dependency is safe here. <see cref="IConfiguration"/> — not
|
||||
/// <c>IClusterRoleInfo</c> — is the source of this node's roles: <c>IClusterRoleInfo</c>'s
|
||||
/// implementation needs the <c>ActorSystem</c>, which does not exist yet at
|
||||
/// <c>ValidateOnStart</c> time.
|
||||
/// constructor dependency is safe here. This validator's only use of configuration is this
|
||||
/// node's roles, so it takes <see cref="EffectiveRoles"/> — the single role authority
|
||||
/// (archreview R3-04) — directly rather than <see cref="IConfiguration"/>.
|
||||
/// </summary>
|
||||
public TelemetryOptionsValidator(IConfiguration configuration)
|
||||
public TelemetryOptionsValidator(EffectiveRoles effectiveRoles)
|
||||
{
|
||||
_configuration = configuration;
|
||||
_effectiveRoles = effectiveRoles;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -118,12 +117,11 @@ public sealed class TelemetryOptionsValidator : IValidateOptions<TelemetryOption
|
||||
+ $"Expected '{TelemetryOptions.ModeDps}' or '{TelemetryOptions.ModeGrpc}'.");
|
||||
}
|
||||
|
||||
// Through ResolveEffectiveRoles so the OTOPCUA_ROLES fallback applies (Gitea #532) — a node
|
||||
// configured only through the env var is still a driver node and must satisfy the Grpc-mode
|
||||
// requirements below.
|
||||
var roles = RoleParser.ResolveEffectiveRoles(
|
||||
_configuration.GetSection("Cluster:Roles").Get<string[]>());
|
||||
var isDriver = Array.IndexOf(roles, RoleParser.Driver) >= 0;
|
||||
// EffectiveRoles is the single role authority (archreview R3-04) — its precedence already
|
||||
// applies the OTOPCUA_ROLES fallback (Gitea #532) and its normalization already folds case, so
|
||||
// a node configured only through the env var, or spelled 'Driver', still satisfies the
|
||||
// Grpc-mode requirements below.
|
||||
var isDriver = _effectiveRoles.HasDriver;
|
||||
|
||||
if (isGrpc)
|
||||
{
|
||||
@@ -137,7 +135,7 @@ public sealed class TelemetryOptionsValidator : IValidateOptions<TelemetryOption
|
||||
+ $"{TelemetryOptions.SectionName}:{nameof(TelemetryOptions.GrpcListenPort)}.");
|
||||
}
|
||||
|
||||
if (roles.Length > 0 && string.IsNullOrEmpty(options.ApiKey))
|
||||
if (_effectiveRoles.Gossiped.Count > 0 && string.IsNullOrEmpty(options.ApiKey))
|
||||
{
|
||||
errors.Add(
|
||||
$"{TelemetryOptions.SectionName}:{nameof(TelemetryOptions.ApiKey)} is empty. Under "
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
@attribute [Authorize(Policy = AdminUiPolicies.AuthenticatedRead)]
|
||||
@rendermode RenderMode.InteractiveServer
|
||||
@using Microsoft.EntityFrameworkCore
|
||||
@using ZB.MOM.WW.OtOpcUa.Cluster
|
||||
@using ZB.MOM.WW.OtOpcUa.Commons.Interfaces
|
||||
@using ZB.MOM.WW.OtOpcUa.Configuration
|
||||
@using ZB.MOM.WW.OtOpcUa.Configuration.Entities
|
||||
@@ -146,7 +147,7 @@ else
|
||||
|
||||
// Union with current Akka driver members so a freshly-joined node that has no
|
||||
// NodeDeploymentState row yet still appears as "waiting".
|
||||
var akkaDrivers = Cluster.MembersWithRole("driver")
|
||||
var akkaDrivers = Cluster.MembersWithRole(RoleParser.Driver)
|
||||
.Select(n => n.Value).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var allNodes = byNode.Keys.Union(akkaDrivers, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase).ToList();
|
||||
@@ -156,7 +157,7 @@ else
|
||||
byNode.TryGetValue(nodeId, out var state);
|
||||
return new NodeRow(
|
||||
NodeId: nodeId,
|
||||
Roles: akkaDrivers.Contains(nodeId) ? new[] { "driver" } : Array.Empty<string>(),
|
||||
Roles: akkaDrivers.Contains(nodeId) ? new[] { RoleParser.Driver } : Array.Empty<string>(),
|
||||
Status: state?.Status,
|
||||
StartedAtUtc: state?.StartedAtUtc,
|
||||
AppliedAtUtc: state?.AppliedAtUtc,
|
||||
|
||||
+4
-2
@@ -181,10 +181,12 @@ public sealed class ClusterNodeAddressReconcilerActor : ReceiveActor, IWithTimer
|
||||
public static IReadOnlyList<(string Host, int Port)> FilterOwnClusterDriverMembers(
|
||||
IEnumerable<Member> members, string? ownClusterRole)
|
||||
{
|
||||
// R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased
|
||||
// 'Driver' (rolling-upgrade window) must not read as a phantom RunningNodeHasNoRow finding.
|
||||
var driverMembers = members
|
||||
.Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(DriverRole));
|
||||
.Where(m => m.Status == MemberStatus.Up && RoleParser.GossipedRolesHave(m.Roles, DriverRole));
|
||||
if (!string.IsNullOrEmpty(ownClusterRole))
|
||||
driverMembers = driverMembers.Where(m => m.Roles.Contains(ownClusterRole));
|
||||
driverMembers = driverMembers.Where(m => RoleParser.GossipedRolesHave(m.Roles, ownClusterRole));
|
||||
return driverMembers
|
||||
.Select(m => (Host: m.Address.Host ?? string.Empty, Port: m.Address.Port ?? 0))
|
||||
.Where(a => !string.IsNullOrWhiteSpace(a.Host) && a.Port > 0)
|
||||
|
||||
@@ -3,6 +3,7 @@ using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy;
|
||||
|
||||
@@ -122,7 +123,9 @@ public sealed class ManualFailoverService : IManualFailoverService
|
||||
}
|
||||
|
||||
private static List<Member> OrderedDrivers(IEnumerable<Member> members) => members
|
||||
.Where(m => m.Status == MemberStatus.Up && m.Roles.Contains(RedundancyStateActor.DriverRole))
|
||||
// R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased
|
||||
// 'Driver' (rolling-upgrade window) must still be a manual-failover candidate.
|
||||
.Where(m => m.Status == MemberStatus.Up && RoleParser.GossipedRolesHave(m.Roles, RedundancyStateActor.DriverRole))
|
||||
.OrderBy(m => m, Member.AgeOrdering)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
@@ -190,7 +190,15 @@ public sealed class RedundancyStateActor : ReceiveActor, IWithTimers
|
||||
if (string.IsNullOrWhiteSpace(host)) continue;
|
||||
|
||||
var isPrimary = driverPrimary is not null && driverPrimary == member.Address;
|
||||
var role = member.Roles.Contains(DriverRole)
|
||||
// R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer running an older build
|
||||
// can still gossip a miscased 'Driver' during a rolling upgrade, and this list is what
|
||||
// /hosts and /fleet render, so a miscased-but-genuinely-driver member must not read as
|
||||
// Detached. NOTE: this does NOT reach SelectDriverPrimary/driverPrimary above — that
|
||||
// delegates to ClusterActiveNode.OldestUpMember (ZB.MOM.WW.Health.Akka), which matches via
|
||||
// Akka's own ordinal Member.HasRole and is not this repo's to change (see R3-04 execution
|
||||
// notes). So a member gossiping only 'Driver' now classifies Secondary rather than
|
||||
// Detached, but a PAIR that BOTH gossip 'Driver' still elects no Primary.
|
||||
var role = RoleParser.GossipedRolesHave(member.Roles, DriverRole)
|
||||
? (isPrimary ? CommonsRedundancyRole.Primary : CommonsRedundancyRole.Secondary)
|
||||
: CommonsRedundancyRole.Detached;
|
||||
|
||||
|
||||
@@ -21,7 +21,9 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane;
|
||||
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
public const string AdminRole = "admin";
|
||||
// R3-04 (theme 4 guard): alias RoleParser.Admin rather than duplicating the literal — the same
|
||||
// pattern Runtime/ServiceCollectionExtensions.DriverRole already follows for RoleParser.Driver.
|
||||
public const string AdminRole = RoleParser.Admin;
|
||||
|
||||
public const string ConfigPublishSingletonName = "config-publish";
|
||||
public const string AdminOperationsSingletonName = "admin-operations";
|
||||
|
||||
@@ -57,13 +57,19 @@ using ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
// Roles drive the entire conditional wiring below — see ZB.MOM.WW.OtOpcUa.Cluster.RoleParser.
|
||||
var roles = RoleParser.Parse(Environment.GetEnvironmentVariable("OTOPCUA_ROLES"));
|
||||
var hasAdmin = roles.Contains("admin");
|
||||
var hasDriver = roles.Contains("driver");
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
// The SINGLE role authority: Cluster:Roles first, OTOPCUA_ROLES as the fallback (archreview
|
||||
// 01/S-12 = 03/C5, R3-04). Resolved off builder.Configuration BEFORE the per-role overlay is
|
||||
// added, because the overlay's FILENAME depends on the answer — which is why a role overlay may
|
||||
// not itself set Cluster:Roles (AkkaClusterOptionsValidator fails start if one ever does).
|
||||
// EffectiveRoles drives the entire conditional wiring below — see ZB.MOM.WW.OtOpcUa.Cluster.
|
||||
var effectiveRoles = EffectiveRoles.Resolve(builder.Configuration);
|
||||
var roles = effectiveRoles.Gossiped;
|
||||
var hasAdmin = effectiveRoles.HasAdmin;
|
||||
var hasDriver = effectiveRoles.HasDriver;
|
||||
builder.Services.AddSingleton(effectiveRoles);
|
||||
|
||||
// Razor class library static assets (_content/<libname>/...) only auto-enable in
|
||||
// the Development environment. Opt in explicitly so the AdminUI's CSS/JS works
|
||||
// regardless of ASPNETCORE_ENVIRONMENT.
|
||||
@@ -77,7 +83,7 @@ builder.Host.UseWindowsService(options => options.ServiceName = "OtOpcUaHost");
|
||||
|
||||
// Per-role appsettings overlay: appsettings.{role}.json (single role) or appsettings.admin-driver.json
|
||||
// (both). Optional — base appsettings.json carries enough to boot if these don't exist.
|
||||
var roleSuffix = roles.Length == 0 ? null : string.Join('-', roles.OrderBy(r => r, StringComparer.Ordinal));
|
||||
var roleSuffix = roles.Count == 0 ? null : string.Join('-', roles.OrderBy(r => r, StringComparer.Ordinal));
|
||||
if (roleSuffix is not null)
|
||||
{
|
||||
builder.Configuration.AddJsonFile($"appsettings.{roleSuffix}.json", optional: true, reloadOnChange: true);
|
||||
@@ -705,8 +711,16 @@ app.MapOtOpcUaMeshGrpcServices(
|
||||
app.MapOtOpcUaHealth();
|
||||
app.MapOtOpcUaMetrics();
|
||||
|
||||
Log.Information("OtOpcUa.Host starting with roles=[{Roles}] (admin={HasAdmin}, driver={HasDriver})",
|
||||
string.Join(",", roles), hasAdmin, hasDriver);
|
||||
Log.Information(
|
||||
"OtOpcUa.Host starting with roles=[{Roles}] (admin={HasAdmin}, driver={HasDriver}) source={Source}",
|
||||
string.Join(",", roles), hasAdmin, hasDriver, effectiveRoles.Source);
|
||||
|
||||
// R3-04 choice 5: Cluster:Roles and OTOPCUA_ROLES set to different normalized role sets is almost
|
||||
// certainly not what the operator intended (Cluster:Roles still wins outright — this is
|
||||
// observability, not a behaviour change). Logged here rather than inside the static factory because
|
||||
// Serilog is not configured yet when EffectiveRoles.Resolve runs at the top of this file.
|
||||
if (effectiveRoles.DisagreementWarning is not null)
|
||||
Log.Warning("{Warning}", effectiveRoles.DisagreementWarning);
|
||||
|
||||
// #561 — startup growth self-check: the banner above went through the live pipeline; verify each
|
||||
// configured file sink physically grew. A sink whose writes silently fail (the wonder-app-vd03
|
||||
|
||||
@@ -5,6 +5,7 @@ using Akka.Event;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
|
||||
@@ -1648,8 +1649,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
if (_driverMemberCountProvider is not null) return _driverMemberCountProvider();
|
||||
try
|
||||
{
|
||||
// R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased
|
||||
// 'Driver' (rolling-upgrade window) must still count toward the R2-04 primary-gate member
|
||||
// count, or a genuine 2-node pair silently takes the single-node allow branch.
|
||||
return Akka.Cluster.Cluster.Get(Context.System).State.Members
|
||||
.Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole));
|
||||
.Count(m => m.Status == Akka.Cluster.MemberStatus.Up
|
||||
&& RoleParser.GossipedRolesHave(m.Roles, ServiceCollectionExtensions.DriverRole));
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Event;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
@@ -338,7 +339,9 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
var isWindowsOnly = driverType is "Galaxy";
|
||||
if (!OperatingSystem.IsWindows() && isWindowsOnly) return true;
|
||||
if (roles.Contains("dev") && isWindowsOnly) return true;
|
||||
// R3-04 (theme 4 guard): GossipedRolesHave, not a bare "dev" literal Contains — the same
|
||||
// case-insensitivity the rest of the role authority gives every other consumer.
|
||||
if (RoleParser.GossipedRolesHave(roles, RoleParser.Dev) && isWindowsOnly) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
@@ -516,8 +517,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
if (_driverMemberCountProvider is not null) return _driverMemberCountProvider();
|
||||
try
|
||||
{
|
||||
// R3-04 (01/C-9): GossipedRolesHave, not ordinal Contains — a peer gossiping a miscased
|
||||
// 'Driver' (rolling-upgrade window) must still count toward the alerts-emit gate.
|
||||
return Akka.Cluster.Cluster.Get(Context.System).State.Members
|
||||
.Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole));
|
||||
.Count(m => m.Status == Akka.Cluster.MemberStatus.Up
|
||||
&& RoleParser.GossipedRolesHave(m.Roles, ServiceCollectionExtensions.DriverRole));
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user