using System.Buffers;
using System.Collections.Concurrent;
using MQTTnet;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Sparkplug;
namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.IntegrationTests;
///
/// Live gate for the Sparkplug primary-host claim (Gitea #511), against the Mosquitto fixture.
///
///
///
/// The issue's acceptance criteria, leg by leg: the broker holds a retained ONLINE while
/// the claim runs; an abrupt death leaves OFFLINE (the Will fires); a clean shutdown
/// leaves OFFLINE; and the pair-node question is answered rather than left to chance —
/// a node that is not Primary claims nothing, and a role change moves the claim.
///
///
/// Each test uses its own host id. A Sparkplug host id is a broker-wide singleton by
/// definition, and the retained STATE outlives the test that wrote it — two tests sharing an id
/// would see each other's leftovers, and would do so intermittently depending on ordering.
///
///
/// Retained messages are cleaned up. A retained ONLINE for a host that no longer
/// exists is exactly the lie this feature prevents, so leaving one on a shared fixture would be
/// poor manners at best and a confusing false positive for the next run at worst.
///
///
[Trait("Category", "Integration")]
[Collection("MqttFixture")]
public sealed class MqttPrimaryHostLiveTests(MqttFixture fixture)
{
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(15);
private void SkipIfUnconfigured() => Assert.SkipWhen(fixture.SkipReason is not null, fixture.SkipReason ?? "");
[Fact]
public async Task A_primary_publishes_a_retained_ONLINE_that_a_late_subscriber_still_sees()
{
// Retention is the point, not an optimisation: an edge node that connects AFTER the host came
// up must still learn the host is alive. A non-retained ONLINE reaches only whoever happened to
// be subscribed at that instant, and every later joiner sees an absent host forever.
SkipIfUnconfigured();
var hostId = NewHostId();
var ct = TestContext.Current.CancellationToken;
await using var host = NewHost(hostId, isPrimary: true);
await host.StartAsync(ct);
host.IsClaimed.ShouldBeTrue("the fixture broker accepted the claim connection");
// Subscribe only NOW — after the birth was published — so only a retained message can satisfy it.
await using var watcher = await StateWatcher.StartAsync(fixture, hostId, ct);
var body = await watcher.WaitForOneAsync(Timeout, ct);
SparkplugStatePayload.TryDecode(body, out var online, out var timestamp).ShouldBeTrue();
online.ShouldBeTrue();
timestamp.ShouldBeGreaterThan(0);
await host.StopAsync(ct);
await ClearRetainedAsync(hostId, ct);
}
[Fact]
public async Task An_abrupt_death_leaves_a_retained_OFFLINE_because_the_will_fires()
{
// THE leg the whole design exists for, and the one no unit test can reach: proof that the
// broker is holding our death certificate and delivers it when the socket dies without a
// goodbye. Without it a crashed node would leave a retained ONLINE asserting forever that a
// dead host is alive — strictly worse than never publishing at all.
SkipIfUnconfigured();
var hostId = NewHostId();
var ct = TestContext.Current.CancellationToken;
var host = NewHost(hostId, isPrimary: true);
await host.StartAsync(ct);
host.IsClaimed.ShouldBeTrue();
await using var watcher = await StateWatcher.StartAsync(fixture, hostId, ct);
var birth = await watcher.WaitForOneAsync(Timeout, ct);
SparkplugStatePayload.TryDecode(birth, out var wasOnline, out var birthTimestamp).ShouldBeTrue();
wasOnline.ShouldBeTrue();
// Kill the socket with no DISCONNECT. A clean teardown would make the broker DISCARD the Will
// by protocol rule, so this test would then be observing the explicit stand-down publish and
// would pass against an implementation that never armed a Will.
await host.AbandonClaimForTests();
var death = await watcher.WaitForMatchAsync(
p => SparkplugStatePayload.TryDecode(p, out var o, out _) && !o, Timeout, ct);
SparkplugStatePayload.TryDecode(death, out _, out var deathTimestamp).ShouldBeTrue();
// Sparkplug 3.0 pairs a host's birth and death by timestamp — an edge node may ignore an
// OFFLINE older than the ONLINE it holds. Mismatched timestamps produce two well-formed
// messages and a death certificate conformant hosts are entitled to throw away.
deathTimestamp.ShouldBe(birthTimestamp);
await host.DisposeAsync();
await ClearRetainedAsync(hostId, ct);
}
[Fact]
public async Task A_clean_shutdown_leaves_a_retained_OFFLINE_without_waiting_for_the_will()
{
SkipIfUnconfigured();
var hostId = NewHostId();
var ct = TestContext.Current.CancellationToken;
var host = NewHost(hostId, isPrimary: true);
await host.StartAsync(ct);
await using var watcher = await StateWatcher.StartAsync(fixture, hostId, ct);
await watcher.WaitForMatchAsync(
p => SparkplugStatePayload.TryDecode(p, out var o, out _) && o, Timeout, ct);
await host.StopAsync(ct);
// Promptly — the assertion is that this arrives well inside the keep-alive window it would take
// the broker to notice a dead socket, which is why the explicit publish exists at all.
await watcher.WaitForMatchAsync(
p => SparkplugStatePayload.TryDecode(p, out var o, out _) && !o, Timeout, ct);
await host.DisposeAsync();
await ClearRetainedAsync(hostId, ct);
}
[Fact]
public async Task A_node_that_is_not_primary_publishes_nothing_at_all()
{
// The pair-node answer. Not merely "does not win the race" — a Secondary must never open a
// claim connection, because the Will it would register is the hazard: its own death would then
// publish OFFLINE over a perfectly healthy Primary's ONLINE and stop the whole group.
SkipIfUnconfigured();
var hostId = NewHostId();
var ct = TestContext.Current.CancellationToken;
await using var watcher = await StateWatcher.StartAsync(fixture, hostId, ct);
await using var host = NewHost(hostId, isPrimary: false);
await host.StartAsync(ct);
host.ClaimAttemptsForTests.ShouldBe(0);
// Give the reconcile loop several ticks to misbehave in.
await Task.Delay(TimeSpan.FromSeconds(3), ct);
host.IsClaimed.ShouldBeFalse();
host.ClaimAttemptsForTests.ShouldBe(0);
watcher.Count.ShouldBe(0, "a Secondary must not touch the STATE topic");
}
[Fact]
public async Task A_failover_moves_the_claim_offline_then_online_again()
{
// Both halves of the accepted trade, observed: losing Primary publishes OFFLINE (a genuine
// state transition — for a window there really is no host), and regaining it republishes
// ONLINE. The republish is the half that would be easy to omit; without it a node that failed
// back would run as Primary while every edge node believed it was gone.
SkipIfUnconfigured();
var hostId = NewHostId();
var ct = TestContext.Current.CancellationToken;
var role = new MutableRole(isPrimary: true);
await using var host = new SparkplugPrimaryHost(Options(hostId), "live-failover", role);
await using var watcher = await StateWatcher.StartAsync(fixture, hostId, ct);
await host.StartAsync(ct);
await watcher.WaitForMatchAsync(
p => SparkplugStatePayload.TryDecode(p, out var o, out _) && o, Timeout, ct);
role.IsPrimary = false;
await watcher.WaitForMatchAsync(
p => SparkplugStatePayload.TryDecode(p, out var o, out _) && !o, Timeout, ct);
await WaitUntilAsync(() => !host.IsClaimed, ct);
var offlineSeen = watcher.CountMatching(
p => SparkplugStatePayload.TryDecode(p, out var o, out _) && !o);
role.IsPrimary = true;
await WaitUntilAsync(() => host.IsClaimed, ct);
// A SECOND online, after the offline — not merely "an online exists", which the birth already
// satisfied before the failover even started.
await WaitUntilAsync(
() => watcher.CountMatching(p => SparkplugStatePayload.TryDecode(p, out var o, out _) && o) >= 2,
ct);
offlineSeen.ShouldBeGreaterThanOrEqualTo(1);
await host.StopAsync(ct);
await ClearRetainedAsync(hostId, ct);
}
// ---- helpers -----------------------------------------------------------------------------------
/// A per-test host id: the retained STATE outlives the test that wrote it.
private static string NewHostId() => $"otopcua-live-{Guid.NewGuid():N}"[..28];
private MqttDriverOptions Options(string hostId) => new()
{
Host = fixture.PlainHost,
Port = fixture.PlainPort,
Username = fixture.Username,
Password = fixture.Password,
UseTls = false,
ClientId = $"otopcua-live-{Guid.NewGuid():N}"[..20],
ConnectTimeoutSeconds = 10,
Mode = MqttMode.SparkplugB,
Sparkplug = new MqttSparkplugOptions
{
GroupId = "OtOpcUaSim",
HostId = hostId,
ActAsPrimaryHost = true,
},
};
private SparkplugPrimaryHost NewHost(string hostId, bool isPrimary) =>
new(Options(hostId), $"live-{hostId}", new MutableRole(isPrimary));
///
/// Clears the retained STATE this test left behind — a zero-length retained publish is MQTT's
/// "forget this". Leaving a retained ONLINE for a host that no longer exists is precisely the
/// lie the feature prevents.
///
private async Task ClearRetainedAsync(string hostId, CancellationToken ct)
{
var client = new MqttClientFactory().CreateMqttClient();
try
{
await client.ConnectAsync(
new MqttClientOptionsBuilder()
.WithTcpServer(fixture.PlainHost, fixture.PlainPort)
.WithCredentials(fixture.Username, fixture.Password)
.WithClientId($"otopcua-live-clean-{Guid.NewGuid():N}"[..23])
.WithTimeout(TimeSpan.FromSeconds(10))
.Build(),
ct);
await client.PublishAsync(
new MqttApplicationMessageBuilder()
.WithTopic(SparkplugTopic.FormatState(hostId))
.WithPayload(Array.Empty())
.WithRetainFlag(true)
.Build(),
ct);
await client.DisconnectAsync(cancellationToken: ct);
}
finally
{
client.Dispose();
}
}
private static async Task WaitUntilAsync(Func condition, CancellationToken ct)
{
var deadline = DateTime.UtcNow + Timeout;
while (!condition())
{
if (DateTime.UtcNow >= deadline)
{
throw new TimeoutException($"Condition was not satisfied within {Timeout}.");
}
await Task.Delay(100, ct);
}
}
/// A role source a test can flip, standing in for a redundancy failover.
private sealed class MutableRole(bool isPrimary) : IDriverPrimaryRoleSource
{
private volatile bool _isPrimary = isPrimary;
public bool IsPrimary
{
get => _isPrimary;
set => _isPrimary = value;
}
public bool ShouldActAsPrimary => _isPrimary;
public void PublishPrimaryRole(bool shouldActAsPrimary) => _isPrimary = shouldActAsPrimary;
}
/// Subscribes one host's STATE topic and records every body seen, in order.
private sealed class StateWatcher : IAsyncDisposable
{
private readonly IMqttClient _client;
private readonly ConcurrentQueue _bodies = new();
private StateWatcher(IMqttClient client) => _client = client;
public int Count => _bodies.Count;
public static async Task StartAsync(MqttFixture fixture, string hostId, CancellationToken ct)
{
var client = new MqttClientFactory().CreateMqttClient();
var watcher = new StateWatcher(client);
client.ApplicationMessageReceivedAsync += args =>
{
watcher._bodies.Enqueue(args.ApplicationMessage.Payload.ToArray());
return Task.CompletedTask;
};
await client.ConnectAsync(
new MqttClientOptionsBuilder()
.WithTcpServer(fixture.PlainHost, fixture.PlainPort)
.WithCredentials(fixture.Username, fixture.Password)
.WithClientId($"otopcua-live-state-{Guid.NewGuid():N}"[..23])
.WithTimeout(TimeSpan.FromSeconds(10))
.Build(),
ct);
await client.SubscribeAsync(
new MqttClientSubscribeOptionsBuilder()
.WithTopicFilter(f => f.WithTopic(SparkplugTopic.FormatState(hostId)))
.Build(),
ct);
return watcher;
}
public int CountMatching(Func predicate) => _bodies.Count(b => Safe(predicate, b));
public async Task WaitForOneAsync(TimeSpan timeout, CancellationToken ct)
=> await WaitForMatchAsync(_ => true, timeout, ct);
public async Task WaitForMatchAsync(Func predicate, TimeSpan timeout, CancellationToken ct)
{
var deadline = DateTime.UtcNow + timeout;
while (true)
{
if (_bodies.FirstOrDefault(b => Safe(predicate, b)) is { } match)
{
return match;
}
if (DateTime.UtcNow >= deadline)
{
throw new TimeoutException(
$"No matching STATE message within {timeout} ({_bodies.Count} seen, none matching).");
}
await Task.Delay(100, ct);
}
}
private static bool Safe(Func predicate, byte[] body)
{
try { return predicate(body); }
catch { return false; }
}
public async ValueTask DisposeAsync()
{
try
{
if (_client.IsConnected)
{
await _client.DisconnectAsync();
}
}
catch
{
// Teardown only.
}
finally
{
_client.Dispose();
}
}
}
}