Files
lmxopcua/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/Sparkplug/SparkplugPrimaryHost.cs
T

528 lines
24 KiB
C#

using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Sparkplug;
/// <summary>
/// Claims the Sparkplug Primary Host Application identity for this node — the three-part contract
/// <see cref="MqttSparkplugOptions.ActAsPrimaryHost"/> promises and, until Gitea #511, did not
/// deliver: a retained <c>ONLINE</c> while the claim is held, a matching retained <c>OFFLINE</c>
/// registered as the MQTT Last Will, and an explicit <c>OFFLINE</c> on a clean stand-down.
/// </summary>
/// <remarks>
/// <para>
/// <b>The claim is a whole dedicated connection, not a message.</b> A Will can only be
/// registered in the CONNECT packet (see <see cref="MqttLastWill"/>), so a claim that must be
/// acquired and released at runtime cannot live on the driver's shared ingest session — that
/// session connects once and stays up, and its Will could be neither added on gaining the role
/// nor withdrawn on losing it. This type therefore opens its <i>own</i> connection when the
/// node gains Primary and closes it when the node loses Primary. The connection's lifetime
/// <i>is</i> the claim, which is what makes every leg of the contract fall out for free:
/// <list type="bullet">
/// <item>the Will is registered because the connection was opened to hold the claim;</item>
/// <item>a <c>kill -9</c> drops the connection, so the broker publishes the Will;</item>
/// <item>a clean stand-down publishes <c>OFFLINE</c> and then disconnects, so the retained
/// message is correct before the Will could ever fire.</item>
/// </list>
/// </para>
/// <para>
/// <b>Why not rebuild the ingest connection instead.</b> That was the shape first sketched on
/// the issue, and it works, but it drops every subscription and the whole birth cache on each
/// role change — producing a data gap on the Secondary→Primary transition, which is precisely
/// the moment a plant most needs values flowing. A second connection costs one extra client id
/// and leaves ingest untouched.
/// </para>
/// <para>
/// <b>Primary-gated, by owner decision (2026-07-31).</b> Both nodes of a redundant pair
/// subscribe independently — MQTT fan-out is per-connection and the Primary gate applies
/// downstream of ingest — so without gating, both would claim the same host id. The sharp
/// failure is not the two fighting over the retained <c>STATE</c>: it is that the
/// <b>Secondary's</b> death would publish <c>OFFLINE</c>, edge nodes would stop publishing,
/// and the surviving Primary would be perfectly healthy throughout. Redundancy would cause the
/// outage the mechanism exists to prevent. The gate is
/// <see cref="IDriverPrimaryRoleSource.ShouldActAsPrimary"/>, sampled on a timer.
/// </para>
/// <para>
/// <b>Accepted trade: a brief edge-visible <c>OFFLINE</c> window during failover</b>, while the
/// old Primary stands down and the new one connects and republishes. That is a genuine state
/// transition — for a window there really is no host — not a glitch to paper over.
/// </para>
/// <para>
/// <b>Reconnect re-asserts the claim.</b> The dedicated connection carries the same supervisor
/// as the ingest one, and <c>Reconnected</c> republishes the birth. Without that, a broker
/// restart would leave the host connected but its retained <c>ONLINE</c> gone — a live host
/// invisible to every edge node, with nothing in any log to say so.
/// </para>
/// </remarks>
public sealed class SparkplugPrimaryHost : IAsyncDisposable
{
/// <summary>
/// Appended to the driver's configured client id for the STATE connection. Two MQTT clients
/// presenting one client id is not reported as an error — the broker silently evicts the older
/// one — so ingest and the host claim would take turns killing each other forever.
/// </summary>
internal const string ClientIdSuffix = "-spbhost";
/// <summary>How often the role gate is sampled. Bounds the failover republish latency.</summary>
private static readonly TimeSpan RoleSampleInterval = TimeSpan.FromSeconds(1);
/// <summary>Ceiling on the delay between failed claim attempts (STAB-27).</summary>
internal static readonly TimeSpan MaxClaimBackoff = TimeSpan.FromSeconds(30);
/// <summary>Consecutive failed claim attempts; reset by a successful claim or a stand-down. Reconcile-loop-owned.</summary>
private int _claimFailures;
private DateTimeOffset _nextClaimAttemptUtc;
private readonly MqttDriverOptions _options;
private readonly string _hostId;
private readonly string _driverId;
private readonly IDriverPrimaryRoleSource _roleSource;
private readonly ILogger? _logger;
private readonly TimeProvider _timeProvider;
/// <summary>
/// Serializes claim/release against each other and against <see cref="DisposeAsync"/>. Never
/// disposed — see <see cref="DisposeAsync"/> for why.
/// </summary>
private readonly SemaphoreSlim _gate = new(1, 1);
private CancellationTokenSource? _loopCts;
/// <summary>The live claim, or null when this node is not currently the primary host.</summary>
private MqttConnection? _connection;
/// <summary>The session bodies backing <see cref="_connection"/>; non-null exactly when it is.</summary>
private SparkplugStateSession? _session;
private int _disposed;
/// <summary>Creates a primary-host claim. Opens nothing — <see cref="StartAsync"/> does that.</summary>
/// <param name="options">Broker connection settings; the host id comes from its Sparkplug section.</param>
/// <param name="driverId">Driver instance id, for log correlation.</param>
/// <param name="roleSource">The Primary data-plane verdict this claim is gated on.</param>
/// <param name="logger">Optional logger.</param>
/// <param name="timeProvider">Clock seam; defaults to <see cref="TimeProvider.System"/>.</param>
/// <exception cref="ArgumentException">
/// The options carry no Sparkplug host id — <see cref="CanClaim"/> exists so a caller never has
/// to hit this.
/// </exception>
public SparkplugPrimaryHost(
MqttDriverOptions options,
string driverId,
IDriverPrimaryRoleSource roleSource,
ILogger? logger = null,
TimeProvider? timeProvider = null)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentException.ThrowIfNullOrWhiteSpace(driverId);
ArgumentNullException.ThrowIfNull(roleSource);
var hostId = options.Sparkplug?.HostId;
if (string.IsNullOrWhiteSpace(hostId))
{
throw new ArgumentException(
"A Sparkplug primary-host claim needs a hostId; check CanClaim first.", nameof(options));
}
_options = options;
_hostId = hostId.Trim();
_driverId = driverId;
_roleSource = roleSource;
_logger = logger;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <summary>
/// Whether <paramref name="options"/> asks for — and can support — a primary-host claim:
/// Sparkplug mode, <c>actAsPrimaryHost</c> set, and a non-blank <c>hostId</c>.
/// </summary>
/// <param name="options">The driver options to test.</param>
/// <returns><see langword="true"/> when a claim should be made.</returns>
/// <remarks>
/// The blank-host-id case is a configuration error the AdminUI form already refuses, but the
/// driver must not fault on it: the config could have been authored before that validation
/// existed. It is reported by <see cref="DescribeUnusable"/> and the flag is treated as off.
/// </remarks>
public static bool CanClaim(MqttDriverOptions options) =>
options is { Mode: MqttMode.SparkplugB, Sparkplug: { ActAsPrimaryHost: true } spb }
&& !string.IsNullOrWhiteSpace(spb.HostId);
/// <summary>
/// Describes why an <c>actAsPrimaryHost</c> request cannot be honoured, or null when it can be
/// (or was never made).
/// </summary>
/// <param name="options">The driver options to test.</param>
/// <returns>An operator-facing reason, or <see langword="null"/>.</returns>
public static string? DescribeUnusable(MqttDriverOptions options)
{
if (options?.Sparkplug is not { ActAsPrimaryHost: true } spb)
{
return null;
}
if (options.Mode != MqttMode.SparkplugB)
{
return "actAsPrimaryHost is set but the driver is not in SparkplugB mode; there is no host state to publish.";
}
return string.IsNullOrWhiteSpace(spb.HostId)
? "actAsPrimaryHost is set but hostId is blank; a host claim has no topic to publish on."
: null;
}
/// <summary>The STATE topic this claim publishes on.</summary>
public string StateTopic => SparkplugTopic.FormatState(_hostId);
/// <summary>
/// Whether the claim is currently held — i.e. a retained <c>ONLINE</c> has been published and a
/// Will is armed behind it.
/// </summary>
public bool IsClaimed => Volatile.Read(ref _connection) is not null;
/// <summary>
/// Test seam: how many times a claim connect has been <i>attempted</i>, successful or not.
/// Always ignored in production.
/// </summary>
/// <remarks>
/// <b>Without this, "a Secondary does not claim" is a vacuous assertion.</b> The obvious test —
/// start a Secondary and assert <see cref="IsClaimed"/> is false — passes identically whether
/// the gate works or whether the gate is deleted and the connect simply fails against a broker
/// the test cannot reach. Counting attempts separates "did not try" from "tried and failed",
/// which is the whole distinction the gate is.
/// </remarks>
internal int ClaimAttemptsForTests => Volatile.Read(ref _claimAttempts);
private int _claimAttempts;
/// <summary>
/// Test seam: kills the claim connection the way a crashed process does — no clean
/// <c>DISCONNECT</c> and no explicit stand-down publish — so the broker delivers the Last Will.
/// See <see cref="MqttConnection.AbandonForTests"/> for why this cannot be reached through the
/// ordinary lifecycle.
/// </summary>
/// <returns>A task completing once the connection is gone.</returns>
internal async Task AbandonClaimForTests()
{
// Stop the reconcile loop first, or the next tick sees no connection, decides the claim is
// lost, and re-claims — republishing ONLINE over the Will this is meant to observe.
var loop = Interlocked.Exchange(ref _loopCts, null);
if (loop is not null)
{
await loop.CancelAsync().ConfigureAwait(false);
loop.Dispose();
}
var connection = Interlocked.Exchange(ref _connection, null);
_session = null;
if (connection is not null)
{
await connection.AbandonForTests().ConfigureAwait(false);
}
}
/// <summary>
/// Starts the role-reconcile loop. Returns once the first reconcile has run, so a caller that
/// starts as Primary has published its <c>ONLINE</c> before this completes.
/// </summary>
/// <param name="cancellationToken">Cancels the initial reconcile.</param>
/// <returns>A task completing after the first reconcile.</returns>
public async Task StartAsync(CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this);
var cts = new CancellationTokenSource();
var previous = Interlocked.Exchange(ref _loopCts, cts);
if (previous is not null)
{
await previous.CancelAsync().ConfigureAwait(false);
previous.Dispose();
}
// Reconcile once inline. A failure here is logged and swallowed, never rethrown: the driver's
// ingest path is the data plane, and refusing to start it because a broker would not accept a
// host claim would trade every tag's value for an advisory message. The loop retries.
await ReconcileAsync(cancellationToken).ConfigureAwait(false);
_ = Task.Run(() => ReconcileLoopAsync(cts.Token), CancellationToken.None);
}
/// <summary>
/// Stands the claim down cleanly: publishes an explicit retained <c>OFFLINE</c>, then
/// disconnects. Idempotent, and safe to call when no claim is held.
/// </summary>
/// <param name="cancellationToken">Bounds the stand-down publish.</param>
/// <returns>A task completing when the claim is released.</returns>
public async Task StopAsync(CancellationToken cancellationToken)
{
var loop = Interlocked.Exchange(ref _loopCts, null);
if (loop is not null)
{
await loop.CancelAsync().ConfigureAwait(false);
loop.Dispose();
}
await ReleaseAsync("shutdown", cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc/>
public async ValueTask DisposeAsync()
{
if (Interlocked.Exchange(ref _disposed, 1) != 0)
{
return;
}
// Bounded: dispose must not hang on an unreachable broker. A stand-down we cannot publish is
// covered by the Will the moment the socket closes, which is the whole point of having one.
using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(_options.ConnectTimeoutSeconds));
try
{
await StopAsync(deadline.Token).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger?.LogDebug(
ex, "MQTT driver '{DriverId}': Sparkplug primary-host stand-down failed during dispose.", _driverId);
}
// _gate is deliberately NOT disposed, matching MqttConnection's rule for its own lifecycle
// primitives. StopAsync cancels the reconcile loop without awaiting it, so a ReconcileAsync can
// still be inside the gate — or on its way to the finally-release — when this returns. Against a
// disposed semaphore that release throws an ObjectDisposedException naming SemaphoreSlim, which
// is logged as a reconcile failure and tells an operator nothing (and hides whatever the
// reconcile was really failing on). Left undisposed the late release is a harmless no-op: the
// gate holds no timer and no registration, and ClaimAsync re-checks _disposed under it, so no
// claim can be created after dispose either way.
}
private async Task ReconcileLoopAsync(CancellationToken ct)
{
try
{
while (!ct.IsCancellationRequested)
{
await Task.Delay(RoleSampleInterval, _timeProvider, ct).ConfigureAwait(false);
await ReconcileAsync(ct).ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
// Shutdown.
}
catch (Exception ex)
{
// The loop must not die silently — a dead loop means the claim freezes in whatever state
// it was last in, including "claimed by a node that is no longer Primary".
_logger?.LogError(
ex, "MQTT driver '{DriverId}': Sparkplug primary-host reconcile loop stopped.", _driverId);
}
}
/// <summary>
/// Brings the held claim into line with the current role verdict. Never throws — a transient
/// broker failure leaves the claim as it was and the next tick tries again.
/// </summary>
private async Task ReconcileAsync(CancellationToken cancellationToken)
{
try
{
if (_roleSource.ShouldActAsPrimary)
{
// STAB-27: a claim that keeps failing backs off instead of redialling (and logging a
// Warning) every RoleSampleInterval for as long as the broker refuses it. The role is still
// sampled each tick — only the connect attempt waits.
if (_claimFailures > 0 && _timeProvider.GetUtcNow() < _nextClaimAttemptUtc)
{
return;
}
await ClaimAsync(cancellationToken).ConfigureAwait(false);
_claimFailures = 0;
}
else
{
// Standing down is not a failure, and a later re-promotion should claim at once.
_claimFailures = 0;
await ReleaseAsync("this node is not the redundancy Primary", cancellationToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
if (_roleSource.ShouldActAsPrimary)
{
_claimFailures++;
var delay = ClaimBackoffFor(_claimFailures);
_nextClaimAttemptUtc = _timeProvider.GetUtcNow() + delay;
// One Warning per ATTEMPT, and attempts are now spaced by the backoff, so this is the
// rate limit: 2 s, 4 s, 8 s … capped at MaxClaimBackoff rather than one line a second.
_logger?.LogWarning(
ex,
"MQTT driver '{DriverId}': could not claim Sparkplug primary host '{HostId}' (attempt {Attempt}); retrying in {RetryIn}.",
_driverId,
_hostId,
_claimFailures,
delay);
}
else
{
_logger?.LogWarning(
ex,
"MQTT driver '{DriverId}': could not reconcile the Sparkplug primary-host claim for '{HostId}'; will retry.",
_driverId,
_hostId);
}
}
}
/// <summary>Capped exponential delay before claim attempt N+1: <c>2^N</c> seconds, at most <see cref="MaxClaimBackoff"/>.</summary>
/// <param name="failures">Consecutive failed claim attempts so far (at least 1).</param>
/// <returns>How long to wait before the next attempt.</returns>
internal static TimeSpan ClaimBackoffFor(int failures)
{
var seconds = Math.Pow(2, Math.Min(failures, 16));
return TimeSpan.FromSeconds(Math.Min(seconds, MaxClaimBackoff.TotalSeconds));
}
private async Task ClaimAsync(CancellationToken cancellationToken)
{
if (Volatile.Read(ref _connection) is not null)
{
return; // Already held; the connection's own supervisor keeps it that way.
}
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_connection is not null || Volatile.Read(ref _disposed) != 0)
{
return;
}
Interlocked.Increment(ref _claimAttempts);
// ONE timestamp for both bodies — the Will is built here, before CONNECT, and the birth
// published after CONNACK. See SparkplugStatePayload's remarks on why they must match.
var session = SparkplugStatePayload.ForSession(
_hostId, _timeProvider.GetUtcNow().ToUnixTimeMilliseconds());
var connection = new MqttConnection(
_options,
_driverId,
_logger,
clientIdSuffix: ClientIdSuffix,
lastWill: new MqttLastWill(
session.Topic, session.OfflineBody, SparkplugStatePayload.StateQos, Retain: true),
// Forced clean, ignoring the driver's configured value: a broker that resumes a prior
// session for this client id may keep the OLD session's Will and discard the one this
// CONNECT carries — leaving a claim with no death certificate behind it, which is the
// single state this whole mechanism exists to make impossible.
cleanSessionOverride: true);
// Re-assert on every reconnect. A broker restart drops retained messages; without this the
// host would be connected and believed offline by every edge node, silently.
connection.Reconnected += async ct =>
{
await connection.PublishAsync(
session.Topic, session.OnlineBody, SparkplugStatePayload.StateQos, retain: true, ct)
.ConfigureAwait(false);
_logger?.LogInformation(
"MQTT driver '{DriverId}': re-published Sparkplug primary-host ONLINE for '{HostId}' after reconnect.",
_driverId,
_hostId);
};
await connection.ConnectAsync(cancellationToken).ConfigureAwait(false);
try
{
await connection.PublishAsync(
session.Topic, session.OnlineBody, SparkplugStatePayload.StateQos, retain: true, cancellationToken)
.ConfigureAwait(false);
}
catch
{
// A connection whose birth never landed is worse than no connection: it holds a Will
// that will publish OFFLINE for a claim that was never asserted, and it looks claimed
// to IsClaimed. Tear it down and let the next tick retry from scratch.
await connection.DisposeAsync().ConfigureAwait(false);
throw;
}
_session = session;
Volatile.Write(ref _connection, connection);
_logger?.LogInformation(
"MQTT driver '{DriverId}': claimed Sparkplug primary host '{HostId}' — retained ONLINE published on '{Topic}', "
+ "OFFLINE armed as the Last Will.",
_driverId,
_hostId,
session.Topic);
}
finally
{
_gate.Release();
}
}
private async Task ReleaseAsync(string reason, CancellationToken cancellationToken)
{
if (Volatile.Read(ref _connection) is null)
{
return;
}
await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
try
{
var connection = Interlocked.Exchange(ref _connection, null);
var session = _session;
_session = null;
if (connection is null || session is null)
{
return;
}
try
{
// Explicit OFFLINE before disconnecting. The Will would eventually say the same thing,
// but only after the broker's keep-alive timeout notices the socket is gone — up to
// 1.5 keep-alive periods of edge nodes believing a host that has already stood down is
// still online. Publishing it makes the transition immediate.
await connection.PublishAsync(
session.Topic, session.OfflineBody, SparkplugStatePayload.StateQos, retain: true, cancellationToken)
.ConfigureAwait(false);
_logger?.LogInformation(
"MQTT driver '{DriverId}': released Sparkplug primary host '{HostId}' ({Reason}) — retained OFFLINE published.",
_driverId,
_hostId,
reason);
}
catch (Exception ex)
{
// Not fatal, and not silent. The Will still covers this: dropping the connection below
// makes the broker publish the identical OFFLINE body. The only thing lost is promptness.
_logger?.LogWarning(
ex,
"MQTT driver '{DriverId}': could not publish the Sparkplug primary-host OFFLINE for '{HostId}'; "
+ "the Last Will will deliver it when the session drops.",
_driverId,
_hostId);
}
await connection.DisposeAsync().ConfigureAwait(false);
}
finally
{
_gate.Release();
}
}
}