ef41a43102
ApplyAndAck advanced _currentRevision and recorded NodeDeploymentState=Applied immediately after ReconcileDrivers, which returns null when the artifact could not be read. So a node that applied NOTHING reported success, claimed a revision whose configuration it never applied, and — because HandleDispatchFromSteady short-circuits on a revision match — could never be healed by re-dispatching that revision. Only a later, different revision recovered it. Now a null blob fails the apply: the revision is left where it was, NodeDeploymentState records Failed with the reason, and the coordinator gets a Failed ACK. Leaving the revision alone is what makes the retry actually land instead of being waved through. This is about telling the truth, not about tearing anything down — the node keeps serving its last-known-good address space, drivers and subscriptions throughout (#485). Tests, RED-first (both verified failing with "should be Failed but was Applied"): the ACK/revision assertion, plus the one that matters — after a failed apply, re-dispatching the SAME revision now genuinely applies. Its recovered artifact adds a second driver precisely so a short-circuited ACK (which has no side effects) cannot satisfy it. Four existing tests changed, and both changes are deliberate: - DriverHostActorTests.SeedDeployment never set ArtifactBlob at all. Its three consumers are about the apply/ACK/state machine, not the artifact, and now need a genuine apply — so the helper seeds a well-formed artifact declaring no drivers. That is what the composer emits for an empty configuration, and is precisely what "bytes we could not read" is NOT. - EmptyArtifact_IsNotCached asserted "the apply still succeeds — an empty artifact is a legitimate no-op deployment". That premise is the bug. Flipped to Failed; the test's actual subject (nothing is cached) is untouched and now holds for two independent reasons. Closes #486 Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
329 lines
18 KiB
C#
329 lines
18 KiB
C#
using System.Text.Json;
|
|
using Akka.Actor;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// Issue #485 (driver-side half) — an artifact the host could not obtain must not be mistaken for a
|
|
/// configuration that contains nothing.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// <c>ReconcileDrivers</c> and <c>PushDesiredSubscriptions</c> both read the artifact as
|
|
/// <c>…FirstOrDefault() ?? Array.Empty<byte>()</c>. The <b>throw</b> path in each already
|
|
/// returns early, but a row that is absent (or carries a zero-length blob) yields empty bytes that
|
|
/// flow onwards as a real answer: zero driver specs, so <c>DriverSpawnPlanner</c> plans every
|
|
/// running child for <c>StopChild</c>, and then an empty desired-subscription set drops each
|
|
/// surviving driver's live handle. A node loses its entire field I/O and still ACKs Applied.
|
|
/// </para>
|
|
/// <para>
|
|
/// The same reasoning as the address-space half applies: nothing legitimate produces a zero-length
|
|
/// blob — an operator who really deletes every driver still deploys a JSON document with empty
|
|
/// arrays — so "no bytes" can only mean the read did not answer. Seeding a row whose
|
|
/// <c>ArtifactBlob</c> is empty reproduces exactly the bytes the missing-row case delivers, without
|
|
/// needing to race a delete against the two reads.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class DriverHostActorUnreadableArtifactTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
|
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
|
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
|
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
|
|
|
|
/// <summary>How long to let post-ACK subscription traffic settle before asserting it did NOT happen.</summary>
|
|
private static readonly TimeSpan SettleWindow = TimeSpan.FromSeconds(2);
|
|
|
|
private const string SpeedRef = "Plant/Modbus/dev1/speed";
|
|
|
|
/// <summary>A dispatch whose artifact reads back as no bytes must leave the running drivers — and their
|
|
/// live subscriptions — exactly as they were.</summary>
|
|
[Fact]
|
|
public void Dispatch_whose_artifact_reads_back_empty_keeps_the_drivers_and_their_subscriptions()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
|
|
// Baseline: the child is up and subscribed to its one raw tag.
|
|
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
|
|
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1");
|
|
|
|
// The next dispatch's artifact comes back as no bytes at all.
|
|
actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout);
|
|
|
|
// The ACK precedes the SubscribeBulk pass, and the child's unsubscribe is a further async self-tell,
|
|
// so settle before asserting an ABSENCE. SettleWindow is calibrated by
|
|
// <see cref="Dropping_a_drivers_last_tag_does_clear_its_subscription"/>, which observes the very same
|
|
// unsubscribe ARRIVING well inside it — without that control this assertion would pass on a race.
|
|
AskDiagnostics(actor); // ordering barrier through the host's mailbox
|
|
Thread.Sleep(SettleWindow);
|
|
|
|
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"); // not stopped
|
|
factory.UnsubscribeCount.ShouldBe(0); // handle not dropped
|
|
factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef);
|
|
}
|
|
|
|
/// <summary>Positive control for the subscription half: a READABLE artifact that keeps the driver but
|
|
/// drops its last tag genuinely does clear the live subscription — the child receives an empty desired
|
|
/// set and unsubscribes. This proves both that the unsubscribe is observable through this factory and
|
|
/// that it lands well within <see cref="SettleWindow"/>, so the absence asserted above is real.</summary>
|
|
[Fact]
|
|
public void Dropping_a_drivers_last_tag_does_clear_its_subscription()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
|
|
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
|
|
factory.UnsubscribeCount.ShouldBe(0);
|
|
|
|
actor.Tell(new DispatchDeployment(SeedTaglessDriverDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout);
|
|
|
|
AwaitAssert(() => factory.UnsubscribeCount.ShouldBe(1), duration: SettleWindow);
|
|
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"); // the driver itself stayed
|
|
}
|
|
|
|
/// <summary>Positive control: the guard keys on "the read gave us nothing", not on "the new config has
|
|
/// fewer drivers". A READABLE artifact that genuinely drops the driver still stops it — otherwise the
|
|
/// test above would pass just as happily against a host that had stopped reconciling altogether.</summary>
|
|
[Fact]
|
|
public void Dispatch_whose_readable_artifact_genuinely_drops_the_driver_still_stops_it()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
|
|
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
|
|
|
|
// A real artifact that simply carries no drivers — an operator deleting the last driver.
|
|
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout);
|
|
|
|
AwaitAssert(
|
|
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
|
|
duration: Timeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Issue #486 — an apply that read no artifact applied nothing, so it must NOT be reported as
|
|
/// Applied. Reporting success also advanced <c>_currentRevision</c>, and because
|
|
/// <c>HandleDispatchFromSteady</c> short-circuits on a revision match, the node could never be
|
|
/// healed by re-dispatching that revision.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Dispatch_whose_artifact_reads_back_empty_acks_Failed_and_leaves_the_revision_alone()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
AskDiagnostics(actor).CurrentRevision.ShouldBe(RevA);
|
|
|
|
actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(Timeout);
|
|
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Failed);
|
|
ack.FailureReason.ShouldNotBeNullOrWhiteSpace();
|
|
|
|
var snapshot = AskDiagnostics(actor);
|
|
snapshot.CurrentRevision.ShouldBe(RevA); // never applied ⇒ never claimed
|
|
snapshot.Drivers.Select(d => d.Name).ShouldContain("drv-1"); // …and #485 still holds
|
|
}
|
|
|
|
/// <summary>Issue #486 — the point of failing the apply: the SAME revision can be dispatched again and
|
|
/// now actually applies, instead of being waved through by the "already at this rev" short-circuit. The
|
|
/// recovered artifact adds a second driver, so a short-circuited ACK (which has no side effects) cannot
|
|
/// satisfy this test — drv-2 only appears if the artifact was genuinely re-read and reconciled.</summary>
|
|
[Fact]
|
|
public void A_failed_apply_is_retried_not_short_circuited_once_the_artifact_is_readable_again()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
|
|
var deploymentId = SeedUnreadableDeployment(db, RevB);
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Failed);
|
|
|
|
// The ConfigDb recovers: the row now carries a real artifact, under the SAME revision.
|
|
SetArtifact(db, deploymentId, TwoDriverArtifact());
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));
|
|
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
AwaitAssert(
|
|
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-2"),
|
|
duration: Timeout);
|
|
AskDiagnostics(actor).CurrentRevision.ShouldBe(RevB);
|
|
}
|
|
|
|
/// <summary>Spawns the host with the subscribing factory, dispatches <paramref name="deploymentId"/> and
|
|
/// waits for the Applied ACK so the child + the initial SubscribeBulk pass are in place.</summary>
|
|
private (IActorRef Actor, Akka.TestKit.TestProbe Coordinator) SpawnHostAndApply(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory factory)
|
|
{
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
driverFactory: factory,
|
|
localRoles: new HashSet<string> { "driver" }));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
return (actor, coordinator);
|
|
}
|
|
|
|
private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
|
|
{
|
|
var probe = CreateTestProbe();
|
|
actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
|
|
return probe.ExpectMsg<NodeDiagnosticsSnapshot>(Timeout);
|
|
}
|
|
|
|
/// <summary>Seeds a Sealed v3 deployment: RawFolder "Plant" → DriverInstance "drv-1" (Modbus, ENABLED so a
|
|
/// real child spawns) → Device "dev1" → Tag "speed" (RawPath <c>Plant/Modbus/dev1/speed</c>).</summary>
|
|
private static DeploymentId SeedV3Deployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
|
|
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
|
DriverInstances = new[]
|
|
{
|
|
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
|
|
},
|
|
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = new[]
|
|
{
|
|
new { TagId = "t-speed", DeviceId = "dev-1", TagGroupId = (string?)null, Name = "speed", DataType = "Double", AccessLevel = 1, TagConfig = "{}" },
|
|
},
|
|
}));
|
|
|
|
/// <summary>A readable artifact carrying drv-1 AND drv-2, so an apply that genuinely happens is
|
|
/// distinguishable from a short-circuited ACK (which spawns nothing).</summary>
|
|
private static byte[] TwoDriverArtifact() => JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
|
DriverInstances = new[]
|
|
{
|
|
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
|
|
new { DriverInstanceId = "drv-2", RawFolderId = "rf-plant", Name = "Modbus2", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
|
|
},
|
|
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = new[]
|
|
{
|
|
new { TagId = "t-speed", DeviceId = "dev-1", TagGroupId = (string?)null, Name = "speed", DataType = "Double", AccessLevel = 1, TagConfig = "{}" },
|
|
},
|
|
});
|
|
|
|
/// <summary>Replaces a seeded deployment's artifact in place — the ConfigDb becoming readable again
|
|
/// without the revision changing.</summary>
|
|
private static void SetArtifact(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, byte[] artifact)
|
|
{
|
|
// ArtifactBlob is init-only, so the row is replaced rather than mutated: delete and re-add under
|
|
// the SAME id + revision. Separate SaveChanges calls so the in-memory provider does not see a
|
|
// delete and an insert of the same key in one change set.
|
|
using var ctx = db.CreateDbContext();
|
|
var row = ctx.Deployments.Single(d => d.DeploymentId == deploymentId.Value);
|
|
var rev = row.RevisionHash;
|
|
ctx.Deployments.Remove(row);
|
|
ctx.SaveChanges();
|
|
|
|
ctx.Deployments.Add(new Deployment
|
|
{
|
|
DeploymentId = deploymentId.Value,
|
|
RevisionHash = rev,
|
|
Status = DeploymentStatus.Sealed,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = DateTime.UtcNow,
|
|
ArtifactBlob = artifact,
|
|
});
|
|
ctx.SaveChanges();
|
|
}
|
|
|
|
/// <summary>Seeds a Sealed deployment whose <c>ArtifactBlob</c> is zero-length — byte-for-byte what the
|
|
/// host's <c>?? Array.Empty<byte>()</c> hands downstream when the row cannot be found.</summary>
|
|
private static DeploymentId SeedUnreadableDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
|
|
SeedDeployment(db, rev, Array.Empty<byte>());
|
|
|
|
/// <summary>Seeds a Sealed deployment carrying a real, readable artifact that keeps drv-1 (so its child
|
|
/// survives the reconcile) but declares no tags for it — the driver's desired set becomes empty.</summary>
|
|
private static DeploymentId SeedTaglessDriverDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
|
|
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
|
DriverInstances = new[]
|
|
{
|
|
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
|
|
},
|
|
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = Array.Empty<object>(),
|
|
}));
|
|
|
|
/// <summary>Seeds a Sealed deployment carrying a real, readable artifact that declares no drivers.</summary>
|
|
private static DeploymentId SeedDriverlessDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
|
|
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = Array.Empty<object>(),
|
|
DriverInstances = Array.Empty<object>(),
|
|
Devices = Array.Empty<object>(),
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = Array.Empty<object>(),
|
|
}));
|
|
|
|
private static DeploymentId SeedDeployment(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev, byte[] artifact)
|
|
{
|
|
var id = DeploymentId.NewId();
|
|
using var ctx = db.CreateDbContext();
|
|
ctx.Deployments.Add(new Deployment
|
|
{
|
|
DeploymentId = id.Value,
|
|
RevisionHash = rev.Value,
|
|
Status = DeploymentStatus.Sealed,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = DateTime.UtcNow,
|
|
ArtifactBlob = artifact,
|
|
});
|
|
ctx.SaveChanges();
|
|
return id;
|
|
}
|
|
|
|
/// <summary>Factory producing one shared <see cref="SubscribableStubDriver"/> for the supported type, so a
|
|
/// REAL (non-stubbed) <see cref="DriverInstanceActor"/> child spawns and its subscribe/unsubscribe traffic
|
|
/// is observable.</summary>
|
|
private sealed class SubscribingDriverFactory(string supportedType) : IDriverFactory
|
|
{
|
|
private readonly SubscribableStubDriver _driver = new();
|
|
|
|
/// <summary>The reference set passed to the driver's most recent <c>SubscribeAsync</c> call.</summary>
|
|
public IReadOnlyList<string>? LastSubscribedRefs => _driver.LastSubscribedRefs;
|
|
|
|
/// <summary>Number of <c>UnsubscribeAsync</c> calls — non-zero means a live handle was torn down.</summary>
|
|
public int UnsubscribeCount => Volatile.Read(ref _driver.UnsubscribeCount);
|
|
|
|
/// <inheritdoc />
|
|
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) =>
|
|
string.Equals(driverType, supportedType, StringComparison.Ordinal) ? _driver : null;
|
|
|
|
/// <inheritdoc />
|
|
public IReadOnlyCollection<string> SupportedTypes => new[] { supportedType };
|
|
}
|
|
}
|