Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorUnreadableArtifactTests.cs
T
Joseph Doherty ef41a43102 fix(drivers): fail the apply when the artifact could not be read (#486)
ApplyAndAck advanced _currentRevision and recorded NodeDeploymentState=Applied
immediately after ReconcileDrivers, which returns null when the artifact could
not be read. So a node that applied NOTHING reported success, claimed a
revision whose configuration it never applied, and — because
HandleDispatchFromSteady short-circuits on a revision match — could never be
healed by re-dispatching that revision. Only a later, different revision
recovered it.

Now a null blob fails the apply: the revision is left where it was,
NodeDeploymentState records Failed with the reason, and the coordinator gets a
Failed ACK. Leaving the revision alone is what makes the retry actually land
instead of being waved through.

This is about telling the truth, not about tearing anything down — the node
keeps serving its last-known-good address space, drivers and subscriptions
throughout (#485).

Tests, RED-first (both verified failing with "should be Failed but was
Applied"): the ACK/revision assertion, plus the one that matters — after a
failed apply, re-dispatching the SAME revision now genuinely applies. Its
recovered artifact adds a second driver precisely so a short-circuited ACK
(which has no side effects) cannot satisfy it.

Four existing tests changed, and both changes are deliberate:

- DriverHostActorTests.SeedDeployment never set ArtifactBlob at all. Its three
  consumers are about the apply/ACK/state machine, not the artifact, and now
  need a genuine apply — so the helper seeds a well-formed artifact declaring
  no drivers. That is what the composer emits for an empty configuration, and
  is precisely what "bytes we could not read" is NOT.
- EmptyArtifact_IsNotCached asserted "the apply still succeeds — an empty
  artifact is a legitimate no-op deployment". That premise is the bug. Flipped
  to Failed; the test's actual subject (nothing is cached) is untouched and now
  holds for two independent reasons.

Closes #486

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
2026-07-21 03:22:40 -04:00

329 lines
18 KiB
C#

using System.Text.Json;
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Issue #485 (driver-side half) — an artifact the host could not obtain must not be mistaken for a
/// configuration that contains nothing.
/// </summary>
/// <remarks>
/// <para>
/// <c>ReconcileDrivers</c> and <c>PushDesiredSubscriptions</c> both read the artifact as
/// <c>…FirstOrDefault() ?? Array.Empty&lt;byte&gt;()</c>. The <b>throw</b> path in each already
/// returns early, but a row that is absent (or carries a zero-length blob) yields empty bytes that
/// flow onwards as a real answer: zero driver specs, so <c>DriverSpawnPlanner</c> plans every
/// running child for <c>StopChild</c>, and then an empty desired-subscription set drops each
/// surviving driver's live handle. A node loses its entire field I/O and still ACKs Applied.
/// </para>
/// <para>
/// The same reasoning as the address-space half applies: nothing legitimate produces a zero-length
/// blob — an operator who really deletes every driver still deploys a JSON document with empty
/// arrays — so "no bytes" can only mean the read did not answer. Seeding a row whose
/// <c>ArtifactBlob</c> is empty reproduces exactly the bytes the missing-row case delivers, without
/// needing to race a delete against the two reads.
/// </para>
/// </remarks>
public sealed class DriverHostActorUnreadableArtifactTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
/// <summary>How long to let post-ACK subscription traffic settle before asserting it did NOT happen.</summary>
private static readonly TimeSpan SettleWindow = TimeSpan.FromSeconds(2);
private const string SpeedRef = "Plant/Modbus/dev1/speed";
/// <summary>A dispatch whose artifact reads back as no bytes must leave the running drivers — and their
/// live subscriptions — exactly as they were.</summary>
[Fact]
public void Dispatch_whose_artifact_reads_back_empty_keeps_the_drivers_and_their_subscriptions()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
// Baseline: the child is up and subscribed to its one raw tag.
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1");
// The next dispatch's artifact comes back as no bytes at all.
actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(Timeout);
// The ACK precedes the SubscribeBulk pass, and the child's unsubscribe is a further async self-tell,
// so settle before asserting an ABSENCE. SettleWindow is calibrated by
// <see cref="Dropping_a_drivers_last_tag_does_clear_its_subscription"/>, which observes the very same
// unsubscribe ARRIVING well inside it — without that control this assertion would pass on a race.
AskDiagnostics(actor); // ordering barrier through the host's mailbox
Thread.Sleep(SettleWindow);
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"); // not stopped
factory.UnsubscribeCount.ShouldBe(0); // handle not dropped
factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef);
}
/// <summary>Positive control for the subscription half: a READABLE artifact that keeps the driver but
/// drops its last tag genuinely does clear the live subscription — the child receives an empty desired
/// set and unsubscribes. This proves both that the unsubscribe is observable through this factory and
/// that it lands well within <see cref="SettleWindow"/>, so the absence asserted above is real.</summary>
[Fact]
public void Dropping_a_drivers_last_tag_does_clear_its_subscription()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
factory.UnsubscribeCount.ShouldBe(0);
actor.Tell(new DispatchDeployment(SeedTaglessDriverDeployment(db, RevB), RevB, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(Timeout);
AwaitAssert(() => factory.UnsubscribeCount.ShouldBe(1), duration: SettleWindow);
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"); // the driver itself stayed
}
/// <summary>Positive control: the guard keys on "the read gave us nothing", not on "the new config has
/// fewer drivers". A READABLE artifact that genuinely drops the driver still stops it — otherwise the
/// test above would pass just as happily against a host that had stopped reconciling altogether.</summary>
[Fact]
public void Dispatch_whose_readable_artifact_genuinely_drops_the_driver_still_stops_it()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
// A real artifact that simply carries no drivers — an operator deleting the last driver.
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(Timeout);
AwaitAssert(
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
duration: Timeout);
}
/// <summary>
/// Issue #486 — an apply that read no artifact applied nothing, so it must NOT be reported as
/// Applied. Reporting success also advanced <c>_currentRevision</c>, and because
/// <c>HandleDispatchFromSteady</c> short-circuits on a revision match, the node could never be
/// healed by re-dispatching that revision.
/// </summary>
[Fact]
public void Dispatch_whose_artifact_reads_back_empty_acks_Failed_and_leaves_the_revision_alone()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
AskDiagnostics(actor).CurrentRevision.ShouldBe(RevA);
actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
var ack = coordinator.ExpectMsg<ApplyAck>(Timeout);
ack.Outcome.ShouldBe(ApplyAckOutcome.Failed);
ack.FailureReason.ShouldNotBeNullOrWhiteSpace();
var snapshot = AskDiagnostics(actor);
snapshot.CurrentRevision.ShouldBe(RevA); // never applied ⇒ never claimed
snapshot.Drivers.Select(d => d.Name).ShouldContain("drv-1"); // …and #485 still holds
}
/// <summary>Issue #486 — the point of failing the apply: the SAME revision can be dispatched again and
/// now actually applies, instead of being waved through by the "already at this rev" short-circuit. The
/// recovered artifact adds a second driver, so a short-circuited ACK (which has no side effects) cannot
/// satisfy this test — drv-2 only appears if the artifact was genuinely re-read and reconciled.</summary>
[Fact]
public void A_failed_apply_is_retried_not_short_circuited_once_the_artifact_is_readable_again()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
var deploymentId = SeedUnreadableDeployment(db, RevB);
actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Failed);
// The ConfigDb recovers: the row now carries a real artifact, under the SAME revision.
SetArtifact(db, deploymentId, TwoDriverArtifact());
actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
AwaitAssert(
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-2"),
duration: Timeout);
AskDiagnostics(actor).CurrentRevision.ShouldBe(RevB);
}
/// <summary>Spawns the host with the subscribing factory, dispatches <paramref name="deploymentId"/> and
/// waits for the Applied ACK so the child + the initial SubscribeBulk pass are in place.</summary>
private (IActorRef Actor, Akka.TestKit.TestProbe Coordinator) SpawnHostAndApply(
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory factory)
{
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
driverFactory: factory,
localRoles: new HashSet<string> { "driver" }));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
return (actor, coordinator);
}
private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
{
var probe = CreateTestProbe();
actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
return probe.ExpectMsg<NodeDiagnosticsSnapshot>(Timeout);
}
/// <summary>Seeds a Sealed v3 deployment: RawFolder "Plant" → DriverInstance "drv-1" (Modbus, ENABLED so a
/// real child spawns) → Device "dev1" → Tag "speed" (RawPath <c>Plant/Modbus/dev1/speed</c>).</summary>
private static DeploymentId SeedV3Deployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
{
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
DriverInstances = new[]
{
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
},
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
TagGroups = Array.Empty<object>(),
Tags = new[]
{
new { TagId = "t-speed", DeviceId = "dev-1", TagGroupId = (string?)null, Name = "speed", DataType = "Double", AccessLevel = 1, TagConfig = "{}" },
},
}));
/// <summary>A readable artifact carrying drv-1 AND drv-2, so an apply that genuinely happens is
/// distinguishable from a short-circuited ACK (which spawns nothing).</summary>
private static byte[] TwoDriverArtifact() => JsonSerializer.SerializeToUtf8Bytes(new
{
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
DriverInstances = new[]
{
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
new { DriverInstanceId = "drv-2", RawFolderId = "rf-plant", Name = "Modbus2", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
},
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
TagGroups = Array.Empty<object>(),
Tags = new[]
{
new { TagId = "t-speed", DeviceId = "dev-1", TagGroupId = (string?)null, Name = "speed", DataType = "Double", AccessLevel = 1, TagConfig = "{}" },
},
});
/// <summary>Replaces a seeded deployment's artifact in place — the ConfigDb becoming readable again
/// without the revision changing.</summary>
private static void SetArtifact(
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, byte[] artifact)
{
// ArtifactBlob is init-only, so the row is replaced rather than mutated: delete and re-add under
// the SAME id + revision. Separate SaveChanges calls so the in-memory provider does not see a
// delete and an insert of the same key in one change set.
using var ctx = db.CreateDbContext();
var row = ctx.Deployments.Single(d => d.DeploymentId == deploymentId.Value);
var rev = row.RevisionHash;
ctx.Deployments.Remove(row);
ctx.SaveChanges();
ctx.Deployments.Add(new Deployment
{
DeploymentId = deploymentId.Value,
RevisionHash = rev,
Status = DeploymentStatus.Sealed,
CreatedBy = "test",
SealedAtUtc = DateTime.UtcNow,
ArtifactBlob = artifact,
});
ctx.SaveChanges();
}
/// <summary>Seeds a Sealed deployment whose <c>ArtifactBlob</c> is zero-length — byte-for-byte what the
/// host's <c>?? Array.Empty&lt;byte&gt;()</c> hands downstream when the row cannot be found.</summary>
private static DeploymentId SeedUnreadableDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
SeedDeployment(db, rev, Array.Empty<byte>());
/// <summary>Seeds a Sealed deployment carrying a real, readable artifact that keeps drv-1 (so its child
/// survives the reconcile) but declares no tags for it — the driver's desired set becomes empty.</summary>
private static DeploymentId SeedTaglessDriverDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
{
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
DriverInstances = new[]
{
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
},
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
TagGroups = Array.Empty<object>(),
Tags = Array.Empty<object>(),
}));
/// <summary>Seeds a Sealed deployment carrying a real, readable artifact that declares no drivers.</summary>
private static DeploymentId SeedDriverlessDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
{
RawFolders = Array.Empty<object>(),
DriverInstances = Array.Empty<object>(),
Devices = Array.Empty<object>(),
TagGroups = Array.Empty<object>(),
Tags = Array.Empty<object>(),
}));
private static DeploymentId SeedDeployment(
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev, byte[] artifact)
{
var id = DeploymentId.NewId();
using var ctx = db.CreateDbContext();
ctx.Deployments.Add(new Deployment
{
DeploymentId = id.Value,
RevisionHash = rev.Value,
Status = DeploymentStatus.Sealed,
CreatedBy = "test",
SealedAtUtc = DateTime.UtcNow,
ArtifactBlob = artifact,
});
ctx.SaveChanges();
return id;
}
/// <summary>Factory producing one shared <see cref="SubscribableStubDriver"/> for the supported type, so a
/// REAL (non-stubbed) <see cref="DriverInstanceActor"/> child spawns and its subscribe/unsubscribe traffic
/// is observable.</summary>
private sealed class SubscribingDriverFactory(string supportedType) : IDriverFactory
{
private readonly SubscribableStubDriver _driver = new();
/// <summary>The reference set passed to the driver's most recent <c>SubscribeAsync</c> call.</summary>
public IReadOnlyList<string>? LastSubscribedRefs => _driver.LastSubscribedRefs;
/// <summary>Number of <c>UnsubscribeAsync</c> calls — non-zero means a live handle was torn down.</summary>
public int UnsubscribeCount => Volatile.Read(ref _driver.UnsubscribeCount);
/// <inheritdoc />
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) =>
string.Equals(driverType, supportedType, StringComparison.Ordinal) ? _driver : null;
/// <inheritdoc />
public IReadOnlyCollection<string> SupportedTypes => new[] { supportedType };
}
}