30d0697c28
`IHostConnectivityProbe` was a dead surface: eleven drivers implement it, `GetHostStatuses()` had ZERO production call sites, and `OnHostStatusChanged` had no subscriber outside the Galaxy driver's own aggregator. Per-host connectivity was computed by every driver and read by nobody. The issue offered "build the publisher or delete it". The publisher as its entity doc described it — driver nodes upserting `DriverHostStatus` rows — is not buildable: per-cluster mesh Phase 4 gates `AddOtOpcUaConfigDb` on the `admin` role, so a driver-only node has no ConfigDb connection to write rows with. So the capability is kept and the transport changed. `DriverHealthChanged.HostStatuses` now carries the probe result to `/hosts` as a Hosts column. That channel already reached the page, already survives the mesh split via the Phase 5 gRPC telemetry stream, and already replays a last-value snapshot on re-subscribe — so per-host state re-primes after a reconnect without a durable store. Both halves of the interface finally do what they are for: the event triggers a prompt publish, the pull is the source of truth. The point of the column is the case the driver-level state chip structurally cannot express: a multi-device driver stays aggregate-Healthy while ONE of its devices is unreachable. Two traps, both pinned by tests that were falsified against the prod code: - The host digest MUST be in the publish fingerprint. On a single-host-down transition every other fingerprint component is unchanged, so the dedup would swallow exactly the publish carrying the news — the trap that already bit the rediscovery signal. Removing it turns the guard test red, verified. - null (no probe) must stay distinct from empty (probe with no hosts). proto3 cannot tell an absent repeated field from an empty one, hence the explicit `has_host_statuses` flag; collapsing them would render every probe-less driver as one whose devices are all fine. Dropped: the DriverHostStatus entity, enum, DbSet, model config and table (migration DropDriverHostStatusTable — empty on every deployment, so the scaffolder's data-loss warning is moot, and Down() recreates it exactly). Found en route, NOT fixed here: `DriverInstanceResilienceStatus` is the identical defect — no writer, no reader, only a DbSet declaration, while the live data rides the `driver-resilience-status` telemetry channel. Its doc-comment now states that rather than describing the sampler and AdminUI join that were never built. Filed as #524 rather than widening this schema change beyond what was asked. Claude-Session: https://claude.ai/code/session_015p7wGqy3YpZNCpDzTpGMKo
251 lines
12 KiB
C#
251 lines
12 KiB
C#
using Akka.Actor;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// Covers the <see cref="IRediscoverable"/> consumer (Gitea #518). Nine drivers raise
|
|
/// <see cref="IRediscoverable.OnRediscoveryNeeded"/> when they observe that a remote's tag set may have
|
|
/// changed — a Galaxy redeploy, a TwinCAT symbol-version bump, an MTConnect agent restart, a Sparkplug
|
|
/// rebirth. Before this wiring NOTHING in <c>src/</c> subscribed, so every one of those raises went into
|
|
/// the void.
|
|
/// <para><b>The signal is advisory.</b> It does not rebuild the served address space: v3 authors raw tags
|
|
/// explicitly through the <c>/raw</c> browse-commit flow, so a runtime graft would create nodes no
|
|
/// operator approved. It rides the driver-health snapshot to the AdminUI as a re-browse prompt.</para>
|
|
/// </summary>
|
|
[Trait("Category", "Unit")]
|
|
public sealed class DriverInstanceActorRediscoverySignalTests : RuntimeActorTestBase
|
|
{
|
|
/// <summary>
|
|
/// The load-bearing case: a driver raises rediscovery while otherwise perfectly healthy, and the
|
|
/// signal reaches the health publisher carrying the driver's reason.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Rediscovery_raise_reaches_the_health_publisher_with_its_reason()
|
|
{
|
|
var driver = new RediscoverableStubDriver();
|
|
var publisher = new RecordingHealthPublisher();
|
|
var parent = CreateTestProbe();
|
|
parent.IgnoreMessages(_ => true);
|
|
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver, healthPublisher: publisher));
|
|
|
|
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
|
AwaitAssert(() => publisher.Published.Count.ShouldBeGreaterThan(0), TimeSpan.FromSeconds(3));
|
|
|
|
driver.RaiseRediscovery("deploy-time-changed");
|
|
|
|
AwaitAssert(
|
|
() =>
|
|
{
|
|
var withSignal = publisher.Published.LastOrDefault(p => p.RediscoveryNeededUtc is not null);
|
|
withSignal.ShouldNotBeNull();
|
|
withSignal!.RediscoveryReason.ShouldBe("deploy-time-changed");
|
|
},
|
|
TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
/// <summary>
|
|
/// <b>Regression guard for the dedup trap.</b> <c>PublishHealthSnapshot</c> suppresses a publish whose
|
|
/// (state, lastSuccessfulRead, lastError, errorCount) tuple repeats. A rediscovery raise on an
|
|
/// otherwise-unchanged Healthy driver leaves all four identical, so unless the rediscovery timestamp
|
|
/// is part of the fingerprint the dedup swallows the very publish carrying the signal and the operator
|
|
/// never sees the prompt.
|
|
/// <para>Positive control: the assertion is that the count STRICTLY INCREASES across the raise. An
|
|
/// "eventually non-null" assertion alone would pass on the warm-up publish and prove nothing. Revert
|
|
/// <c>_rediscoveryNeededUtc</c> out of the fingerprint tuple and this test must fail.</para>
|
|
/// </summary>
|
|
[Fact]
|
|
public void Rediscovery_raise_is_not_swallowed_by_the_unchanged_health_dedup()
|
|
{
|
|
var driver = new RediscoverableStubDriver();
|
|
var publisher = new RecordingHealthPublisher();
|
|
var parent = CreateTestProbe();
|
|
parent.IgnoreMessages(_ => true);
|
|
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver, healthPublisher: publisher));
|
|
|
|
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
|
AwaitAssert(() => publisher.Published.Count.ShouldBeGreaterThan(0), TimeSpan.FromSeconds(3));
|
|
|
|
// Let the actor settle so nothing else is in flight, then capture the baseline. Everything about
|
|
// the driver's health is now stable — only the rediscovery flag will change.
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
|
var before = publisher.Published.Count;
|
|
|
|
driver.RaiseRediscovery("symbol-version-changed");
|
|
|
|
AwaitAssert(
|
|
() => publisher.Published.Count.ShouldBeGreaterThan(before),
|
|
TimeSpan.FromSeconds(3));
|
|
publisher.Published[^1].RediscoveryNeededUtc.ShouldNotBeNull();
|
|
}
|
|
|
|
/// <summary>
|
|
/// The signal is sticky: a later health publish still carries it. The remote's tag set stayed
|
|
/// changed, and only an operator re-browsing resolves that — which this actor cannot observe. A
|
|
/// flag that cleared itself on the next heartbeat would be a prompt the operator could miss entirely.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Rediscovery_flag_persists_on_subsequent_health_publishes()
|
|
{
|
|
var driver = new RediscoverableStubDriver();
|
|
var publisher = new RecordingHealthPublisher();
|
|
var parent = CreateTestProbe();
|
|
parent.IgnoreMessages(_ => true);
|
|
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver, healthPublisher: publisher));
|
|
|
|
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
|
AwaitAssert(() => publisher.Published.Count.ShouldBeGreaterThan(0), TimeSpan.FromSeconds(3));
|
|
driver.RaiseRediscovery("agent-instance-changed");
|
|
AwaitAssert(
|
|
() => publisher.Published.Any(p => p.RediscoveryNeededUtc is not null),
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Force a further publish by changing something else about the driver's health.
|
|
driver.SetLastError("transient read failure");
|
|
AwaitAssert(
|
|
() => publisher.Published.Any(p => p.LastError == "transient read failure"),
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
var latest = publisher.Published[^1];
|
|
latest.RediscoveryNeededUtc.ShouldNotBeNull();
|
|
latest.RediscoveryReason.ShouldBe("agent-instance-changed");
|
|
}
|
|
|
|
/// <summary>
|
|
/// <b>Leak guard.</b> The <see cref="IDriver"/> instance can OUTLIVE the actor — the host stops a
|
|
/// child and respawns another around the same driver object — so a missing <c>-=</c> in
|
|
/// <c>PostStop</c> accumulates one handler per respawn, each holding a dead <c>Self</c>. Asserts the
|
|
/// driver's invocation list is empty again after the actor stops.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Stopping_the_actor_detaches_the_rediscovery_handler()
|
|
{
|
|
var driver = new RediscoverableStubDriver();
|
|
var parent = CreateTestProbe();
|
|
parent.IgnoreMessages(_ => true);
|
|
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver));
|
|
|
|
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
|
AwaitAssert(() => driver.SubscriberCount.ShouldBe(1), TimeSpan.FromSeconds(3));
|
|
|
|
Watch(actor);
|
|
actor.Tell(PoisonPill.Instance);
|
|
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
|
|
|
|
driver.SubscriberCount.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>A driver that does NOT implement <see cref="IRediscoverable"/> must publish a null flag —
|
|
/// the field is absent, not defaulted to "needs re-browse".</summary>
|
|
[Fact]
|
|
public void A_non_rediscoverable_driver_never_sets_the_flag()
|
|
{
|
|
var publisher = new RecordingHealthPublisher();
|
|
var parent = CreateTestProbe();
|
|
parent.IgnoreMessages(_ => true);
|
|
var actor = parent.ChildActorOf(DriverInstanceActor.Props(new StubDriver(), healthPublisher: publisher));
|
|
|
|
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
|
AwaitAssert(() => publisher.Published.Count.ShouldBeGreaterThan(0), TimeSpan.FromSeconds(3));
|
|
|
|
publisher.Published.ShouldAllBe(p => p.RediscoveryNeededUtc == null);
|
|
}
|
|
|
|
/// <summary>Captures every health publish so a test can assert on the rediscovery fields.</summary>
|
|
private sealed record HealthPublish(
|
|
string ClusterId,
|
|
string DriverInstanceId,
|
|
DriverHealth Health,
|
|
int ErrorCount5Min,
|
|
DateTime? RediscoveryNeededUtc,
|
|
string? RediscoveryReason)
|
|
{
|
|
public string? LastError => Health.LastError;
|
|
}
|
|
|
|
private sealed class RecordingHealthPublisher : IDriverHealthPublisher
|
|
{
|
|
private readonly List<HealthPublish> _published = [];
|
|
|
|
/// <summary>Thread-safe snapshot — <c>Publish</c> is called from the actor thread while the test
|
|
/// asserts from its own.</summary>
|
|
public IReadOnlyList<HealthPublish> Published
|
|
{
|
|
get { lock (_published) return _published.ToArray(); }
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void Publish(
|
|
string clusterId,
|
|
string driverInstanceId,
|
|
DriverHealth health,
|
|
int errorCount5Min,
|
|
DateTime? rediscoveryNeededUtc = null,
|
|
string? rediscoveryReason = null,
|
|
IReadOnlyList<HostConnectivityStatus>? hostStatuses = null)
|
|
{
|
|
lock (_published)
|
|
{
|
|
_published.Add(new HealthPublish(
|
|
clusterId, driverInstanceId, health, errorCount5Min, rediscoveryNeededUtc, rediscoveryReason));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// A stub driver exposing <see cref="IRediscoverable"/>, a hook to raise it, and the live subscriber
|
|
/// count (for the detach guard).
|
|
/// <para><b>Implemented from scratch rather than derived from the shared <c>StubDriver</c> on
|
|
/// purpose.</b> That one returns <c>GetHealth() => new(Healthy, DateTime.UtcNow, null)</c>, so its
|
|
/// health fingerprint differs on EVERY call and the dedup under test never engages — which would make
|
|
/// <see cref="Rediscovery_raise_is_not_swallowed_by_the_unchanged_health_dedup"/> pass vacuously
|
|
/// whether or not the fix is present. This stub's health is STABLE until a test changes it.</para>
|
|
/// </summary>
|
|
private sealed class RediscoverableStubDriver : IDriver, IRediscoverable
|
|
{
|
|
private static readonly DateTime FixedLastRead = new(2026, 7, 27, 12, 0, 0, DateTimeKind.Utc);
|
|
private volatile string? _lastError;
|
|
|
|
/// <inheritdoc />
|
|
public event EventHandler<RediscoveryEventArgs>? OnRediscoveryNeeded;
|
|
|
|
/// <inheritdoc />
|
|
public string DriverInstanceId => "rediscoverable-stub-1";
|
|
|
|
/// <inheritdoc />
|
|
public string DriverType => "Stub";
|
|
|
|
/// <summary>Number of live subscribers on <see cref="OnRediscoveryNeeded"/>.</summary>
|
|
public int SubscriberCount => OnRediscoveryNeeded?.GetInvocationList().Length ?? 0;
|
|
|
|
/// <summary>Raises the event exactly as a real driver's watcher does.</summary>
|
|
public void RaiseRediscovery(string reason)
|
|
=> OnRediscoveryNeeded?.Invoke(this, new RediscoveryEventArgs(reason, ScopeHint: null));
|
|
|
|
/// <summary>Changes the reported health so a test can force a publish that is NOT the rediscovery one.</summary>
|
|
public void SetLastError(string error) => _lastError = error;
|
|
|
|
/// <inheritdoc />
|
|
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <inheritdoc />
|
|
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <inheritdoc />
|
|
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <inheritdoc />
|
|
public DriverHealth GetHealth() => new(DriverState.Healthy, FixedLastRead, _lastError);
|
|
|
|
/// <inheritdoc />
|
|
public long GetMemoryFootprint() => 0;
|
|
|
|
/// <inheritdoc />
|
|
public Task FlushOptionalCachesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
|
}
|
|
}
|