Merge #867 Heidenhain integer leaves keep their declared type; #868 ConfigPublishCoordinator survives ConfigDb outage at start

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 12:29:00 -05:00
11 changed files with 720 additions and 41 deletions
@@ -0,0 +1,133 @@
using System.Buffers.Binary;
using System.Text;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests.Simulator;
namespace ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests;
/// <summary>
/// lmxopcua#867: the published value's RUNTIME type must be the CLR type of the leaf's declared data type.
/// Shouldly equality cannot catch this (7.0 equals 7L), so every assertion here compares <c>GetType()</c>.
/// </summary>
[Trait("Category", "Unit")]
public sealed class HeidenhainLeafRuntimeTypeTests
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static Type ClrType(DriverDataType type, bool isArray)
{
var scalar = type switch
{
DriverDataType.Boolean => typeof(bool),
DriverDataType.Int16 => typeof(short),
DriverDataType.Int32 => typeof(int),
DriverDataType.Int64 => typeof(long),
DriverDataType.Float64 => typeof(double),
DriverDataType.String => typeof(string),
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null),
};
return isArray ? scalar.MakeArrayType() : scalar;
}
/// <summary>Every fixed value leaf (capability strings excluded), with its declared type.</summary>
public static TheoryData<string> CatalogValueLeaves()
{
var data = new TheoryData<string>();
foreach (var e in HeidenhainCatalog.Entries.Where(e => !e.IsCapability)) data.Add(e.Address);
return data;
}
private static SimulatedControl FullyPopulatedControl()
{
var c = new SimulatedControl();
c.Runtime[22] = () => SimReply.Ok("S_RI", [0, 1, .. Encoding.ASCII.GetBytes("1.5\0X\0")]);
c.Runtime[23] = () => SimReply.Ok("S_RI", [0, 4]);
c.Runtime[26] = () => SimReply.Ok("S_RI", [0, 7]);
c.Runtime[24] = () => SimReply.Ok("S_RI", [0, 0, 0, 5, .. "A\0B\0"u8.ToArray()]);
c.Runtime[25] = () => SimReply.Ok("S_RI", new byte[12]);
var alarm = new byte[10];
BinaryPrimitives.WriteInt16BigEndian(alarm, 3);
BinaryPrimitives.WriteInt16BigEndian(alarm.AsSpan(2), 4);
BinaryPrimitives.WriteInt32BigEndian(alarm.AsSpan(4), 12);
alarm[8] = 65;
c.Runtime[27] = () => SimReply.Ok("S_RI", alarm);
c.Runtime[28] = () => SimReply.Refuse(57);
// 24-byte tool reply: number 49, index 1, axis 2, length 100.5, radius 3.25 (float64le).
var tool = new byte[24];
BinaryPrimitives.WriteUInt32BigEndian(tool, 49);
BinaryPrimitives.WriteUInt16BigEndian(tool.AsSpan(4), 1);
BinaryPrimitives.WriteUInt16BigEndian(tool.AsSpan(6), 2);
BinaryPrimitives.WriteDoubleLittleEndian(tool.AsSpan(8), 100.5);
BinaryPrimitives.WriteDoubleLittleEndian(tool.AsSpan(16), 3.25);
c.Runtime[51] = () => SimReply.Ok("S_RI", tool);
for (var s = 30; s <= 34; s++) c.Runtime[s] = () => SimReply.Ok("S_RI", [0, 0, 1, 0]);
return c;
}
[Theory]
[MemberData(nameof(CatalogValueLeaves))]
public async Task Catalog_leaf_publishes_the_clr_type_of_its_declared_data_type(string address)
{
HeidenhainCatalog.TryGet(address, out var entry).ShouldBeTrue();
var control = FullyPopulatedControl();
await using var sim = Lsv2Simulator.WithControl(control);
var client = new Lsv2.Lsv2Client("127.0.0.1", sim.Port, TimeSpan.FromSeconds(1));
var tag = TestTags.Def(address);
var settings = new HeidenhainSessionSettings(true, true, false, 64, true, false, false);
await using var session = new HeidenhainSession(client, settings, (_, _) => ValueTask.FromResult(""));
await client.OpenAsync(Ct);
await session.InitializeAsync(Ct);
var sample = (await session.ReadBatchAsync([tag], Ct))[0];
sample.Status.ShouldBe(HeidenhainStatus.Good, address);
sample.Value.ShouldNotBeNull(address);
sample.Value.GetType().ShouldBe(ClrType(entry!.DataType, entry.IsArray), address);
control.Violations.ShouldBeEmpty();
}
[Fact]
public void Every_integer_catalog_leaf_is_covered()
{
// Pins the theory's reach: the integer leaves #867 broke are in the data set.
var leaves = CatalogValueLeaves().Select(r => r.Data).ToList();
leaves.ShouldContain("Tool/Number");
leaves.ShouldContain("Tool/Index");
leaves.ShouldContain("Tool/AxisCode");
leaves.ShouldContain("Alarms/Count");
leaves.ShouldContain("Program/Line");
leaves.ShouldContain("Status/ExecutionState");
leaves.Count(l => l.StartsWith("Timers/", StringComparison.Ordinal)).ShouldBe(HeidenhainCatalog.TimerNames.Count);
}
[Theory]
[InlineData(DriverDataType.Int16, 42)]
[InlineData(DriverDataType.Int32, 42)]
[InlineData(DriverDataType.Int64, 42)]
[InlineData(DriverDataType.Int16, 42L)]
[InlineData(DriverDataType.Int32, 42L)]
[InlineData(DriverDataType.Int64, 42L)]
[InlineData(DriverDataType.Int16, 42.0)]
[InlineData(DriverDataType.Int32, 42.0)]
[InlineData(DriverDataType.Int64, 42.0)]
[InlineData(DriverDataType.Boolean, true)]
[InlineData(DriverDataType.Boolean, false)]
public void Binding_coercion_publishes_the_clr_type_of_the_declared_data_type(DriverDataType type, object native)
{
HeidenhainBindingCodec.Coerce(native, type).GetType().ShouldBe(ClrType(type, false));
}
[Theory]
[InlineData(HeidenhainPlcArea.Byte, new byte[] { 7 }, DriverDataType.Int16)]
[InlineData(HeidenhainPlcArea.Byte, new byte[] { 7 }, DriverDataType.Int32)]
[InlineData(HeidenhainPlcArea.Word, new byte[] { 7, 0 }, DriverDataType.Int64)]
[InlineData(HeidenhainPlcArea.Dword, new byte[] { 7, 0, 0, 0 }, DriverDataType.Int32)]
[InlineData(HeidenhainPlcArea.InputWord, new byte[] { 7, 0 }, DriverDataType.Int32)]
[InlineData(HeidenhainPlcArea.Marker, new byte[] { 1 }, DriverDataType.Boolean)]
public void Plc_binding_decode_publishes_the_clr_type_of_the_declared_data_type(HeidenhainPlcArea area, byte[] bytes, DriverDataType type)
{
HeidenhainBindingCodec.DecodePlc(area, bytes, type).GetType().ShouldBe(ClrType(type, false));
}
}
@@ -0,0 +1,185 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// lmxopcua#868: the coordinator's recovery read used to run synchronously in <c>PreStart</c>. With the
/// ConfigDb unreachable when the singleton started, the actor died on initialization, was never re-created,
/// and every deployment hung. It now loads asynchronously with backoff, queues what arrives meanwhile and
/// reports <see cref="ConfigPublishCoordinatorState.Loading"/> until the load succeeds.
/// </summary>
public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTestBase
{
private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('d', 64));
private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);
private static readonly TimeSpan RetryMin = TimeSpan.FromMilliseconds(50);
private static readonly TimeSpan RetryMax = TimeSpan.FromMilliseconds(200);
private const string Node = "site-a-1:4053";
[Fact]
public void Db_unreachable_at_start_does_not_kill_the_coordinator_and_a_deploy_sent_during_the_outage_completes_once_it_returns()
{
var dbName = Guid.NewGuid().ToString("N");
var direct = NewInMemoryDbFactory(dbName);
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
SeedNodes(direct, Node);
var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
var readiness = new ConfigPublishCoordinatorReadiness();
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
Watch(coordinator);
// Several failed loads: the actor keeps retrying (it is alive) and reports Loading with the error.
AwaitAssert(() =>
{
outage.Attempts.ShouldBeGreaterThanOrEqualTo(3);
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Loading);
readiness.Current.FailedLoadAttempts.ShouldBeGreaterThanOrEqualTo(2);
readiness.Current.LastError.ShouldNotBeNull();
readiness.Current.LastError.ShouldContain("unreachable");
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
// A deployment dispatched during the outage is queued, not processed and not dropped.
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
using (var db = direct.CreateDbContext())
{
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching);
db.NodeDeploymentStates.ShouldBeEmpty();
}
// The DB comes back: the load succeeds, the queued dispatch runs, and the node's ack seals it.
outage.Down = false;
AwaitAssert(() =>
{
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
using var db = direct.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { Node });
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = direct.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
ExpectNoMsg(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); // never Terminated
}
[Fact]
public void In_flight_deployment_is_recovered_after_the_outage_and_an_ack_queued_during_it_is_not_lost()
{
var dbName = Guid.NewGuid().ToString("N");
var direct = NewInMemoryDbFactory(dbName);
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
SeedNodes(direct, Node, "site-a-2:4053");
var deploymentId = SeedDeployment(direct, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow);
using (var db = direct.CreateDbContext())
{
db.NodeDeploymentStates.Add(new NodeDeploymentState
{ NodeId = Node, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying });
db.NodeDeploymentStates.Add(new NodeDeploymentState
{ NodeId = "site-a-2:4053", DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied, AppliedAtUtc = DateTime.UtcNow });
db.SaveChanges();
}
var readiness = new ConfigPublishCoordinatorReadiness();
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(2),
duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
// The last outstanding ack arrives while the coordinator is still loading: it is queued, then applied
// against the recovered expected set (site-a-2 already Applied in the DB) and seals the deployment.
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
outage.Down = false;
AwaitAssert(() =>
{
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
using var db = direct.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
[Fact]
public void Readiness_is_NotHosted_after_the_coordinator_stops()
{
var readiness = new ConfigPublishCoordinatorReadiness();
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(NewInMemoryDbFactory(), NoTimeout, readiness: readiness));
AwaitAssert(() => readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready),
duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
Watch(coordinator);
Sys.Stop(coordinator);
ExpectTerminated(coordinator, cancellationToken: TestContext.Current.CancellationToken);
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.NotHosted);
}
private static DeploymentId SeedDeployment(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentStatus status, DateTime createdAtUtc)
{
var id = DeploymentId.NewId();
using var db = dbFactory.CreateDbContext();
db.Deployments.Add(new Deployment
{
DeploymentId = id.Value,
RevisionHash = TestRevision.Value,
Status = status,
CreatedBy = "test",
CreatedAtUtc = createdAtUtc,
});
db.SaveChanges();
return id;
}
private static void SeedNodes(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] nodeIds)
{
using var db = dbFactory.CreateDbContext();
db.ServerClusters.Add(new ServerCluster
{
ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
});
foreach (var nodeId in nodeIds)
{
db.ClusterNodes.Add(new ClusterNode
{
NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0],
ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test",
});
}
db.SaveChanges();
}
/// <summary>A factory that throws while <see cref="Down"/> — a ConfigDb that is unreachable.</summary>
private sealed class OutageDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner) : IDbContextFactory<OtOpcUaConfigDbContext>
{
private volatile bool _down;
private int _attempts;
public bool Down { get => _down; set => _down = value; }
public int Attempts => Volatile.Read(ref _attempts);
public OtOpcUaConfigDbContext CreateDbContext()
{
Interlocked.Increment(ref _attempts);
if (_down) throw new InvalidOperationException("ConfigDb unreachable (simulated outage)");
return inner.CreateDbContext();
}
}
}
@@ -136,9 +136,9 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout));
var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty<IDriverProbe>()));
// Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and
// the actor would be stopped, not restarted). Identify is processed after creation, so a
// resolved selection proves PreStart completed.
// Arm the fault AFTER PreStart has run. Since lmxopcua#868 the recovery read is asynchronous and
// retried, so a fault that lands on it instead of on HandleDispatch is also recovered (the dispatch is
// queued until the retried load succeeds) — either way the deployment must complete.
await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget, TestContext.Current.CancellationToken);
faulting.FailNext();
admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
@@ -0,0 +1,65 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.Host.Health;
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health;
/// <summary>lmxopcua#868: the deploy coordinator reports Degraded (never Unhealthy) while it cannot load from the ConfigDb.</summary>
public sealed class ConfigPublishCoordinatorHealthCheckTests
{
private static async Task<HealthCheckResult> Check(ConfigPublishCoordinatorReadiness? readiness) =>
await new ConfigPublishCoordinatorHealthCheck(readiness).CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
[Fact]
public async Task Loading_after_failed_attempts_is_degraded_and_names_the_error()
{
var r = new ConfigPublishCoordinatorReadiness();
r.MarkLoading(3, "connection refused");
var result = await Check(r);
result.Status.ShouldBe(HealthStatus.Degraded);
result.Description.ShouldNotBeNull();
result.Description.ShouldContain("connection refused");
result.Description.ShouldContain("queued");
}
[Fact]
public async Task Ready_is_healthy()
{
var r = new ConfigPublishCoordinatorReadiness();
r.MarkLoading(2, "x");
r.MarkReady();
(await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
}
[Fact]
public async Task Not_hosted_and_untracked_are_healthy()
{
(await Check(new ConfigPublishCoordinatorReadiness())).Status.ShouldBe(HealthStatus.Healthy);
(await Check(null)).Status.ShouldBe(HealthStatus.Healthy);
}
[Theory]
[InlineData(true, true)]
[InlineData(false, false)]
public void It_is_registered_on_the_ready_tier_only_on_admin_nodes(bool hasAdmin, bool registered)
{
var registrations = new ServiceCollection()
.AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: true)
.BuildServiceProvider()
.GetRequiredService<IOptions<HealthCheckServiceOptions>>()
.Value.Registrations
.Where(x => x.Name == ConfigPublishCoordinatorHealthCheck.Name)
.ToList();
if (!registered)
{
registrations.ShouldBeEmpty();
return;
}
registrations.ShouldHaveSingleItem().Tags.ShouldBe([ZbHealthTags.Ready]);
}
}