97dc715dac
v2-ci / unit-tests (push) Has been cancelled
Three write bounds were configured independently: the inbound-write gateway's 10 s Ask, DriverHostActor's fixed 8 s Ask of the child, and the child's own write timeout (5 s + up to 2 s grace) with a fixed 8 s queue budget. An outer Ask that fires first reports the write failed and the node reverts while the child is still driving the device, so any write timeout above ~6 s (or a write dequeued late in its 8 s budget) let the device take a value the client was told failed. DriverWriteBounds (Runtime) derives them all from DriverWrites:Timeout: token cancel = Timeout abandon = Timeout + min(Timeout/2, 2 s) (RunBound) queue budget = RunBound child replies <= queue budget + RunBound (ReplyBound) host Ask = ReplyBound + 1 s gateway Ask = host Ask + 1 s Default Timeout is 10 s (the Tier C Write timeout, so no tier default is capped below its own policy); host Ask 25 s, gateway Ask 26 s. Tier A/B writes are still cancelled first by their 2 s / 4 s tier timeouts. - DriverHostActor takes writeBounds + writeClock, spawns children with them, and runs its Ask deadline as a token on the write clock (Infinite Akka timeout), so host and child bounds share one clock. - WriteAttribute gains IssuedAt (write-clock timestamp, stamped by the host): the queue budget counts mailbox time too, and a write past it is failed BadTimeout and never started - including on arrival. Stale queued writes are expired across the whole queue, not only up to a free slot. - ActorNodeWriteGateway's default Ask = Default.GatewayAsk; the hosted service passes the configured one. - Startup validation (Host DriverWriteBoundsBinding, same warn-and-clamp style as ScriptingOptionsBinding): unparsable / non-positive -> default, above 5 min -> clamped, and each tier whose default Write timeout exceeds the write timeout is named in a Warning (its policy would never take effect). The derived bounds are logged at Information. Per-instance ResilienceConfig Write overrides arrive with a deployment, so DriverCapabilityInvokerFactory warns at spawn (writeTimeoutCeiling). - appsettings.json documents DriverWrites:Timeout. Tests (manual clock): - DriverHostActorWriteAskTests: a held write under a 9 s timeout survives an 8.5 s step and succeeds; a token-ignoring write is answered by the child's BadTimeout at 9 s + grace, not by the Ask; the Infinite-timeout Ask ends exactly at its clock deadline. - DriverInstanceActorConcurrentWriteTests: a write issued longer ago than the queue budget is failed and never reaches the driver; positive control at exactly the budget is written. - DriverWriteBoundsTests: every outer bound covers the inner ones for 0.5 s to 300 s timeouts. - DriverWriteBoundsBindingTests: default fits every tier; 3 s timeout is caught at startup with Tier B and C named (not A); bad values fall back with a warning; over-max clamps. - DriverCapabilityInvokerFactoryTests: spawn warns when a Write override exceeds the ceiling; no warning within it or without a ceiling. Mutation checks: a fixed 8 s host Ask on the same clock fails both 9 s host tests; stamping arrival time instead of IssuedAt fails the issued-at test; dropping the tier check / factory check fails their tests. Runtime.Tests 961 passed / 3 skipped at AKKA_TEST_TIMEFACTOR 3 and 1; Host.Tests 232/232 at 3 and 1; Core.Tests 361 passed / 1 skipped. Solution Release build: 0 warnings. ContinuousHistorizationRecorderTypedValueTests (historian) deliberately left for the historian lane. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
240 lines
11 KiB
C#
240 lines
11 KiB
C#
using System.Text.Json;
|
|
using Akka.Actor;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// lmxopcua#777: <see cref="DriverHostActor"/>'s write Ask is derived from the configured write timeout
|
|
/// (<see cref="DriverWriteBounds.HostAsk"/>) rather than a fixed 8 s, so a write the child is still entitled to run
|
|
/// is never reported failed by the Ask — the node would be reverted while the device could still take the value.
|
|
/// <para>
|
|
/// Drives a real apply (a real <see cref="DriverInstanceActor"/> child), then routes a held write with the
|
|
/// host and the child on ONE manual clock: the host's Ask and the child's bounds both run on it, so stepping
|
|
/// the clock shows which one answers first.
|
|
/// </para>
|
|
/// </summary>
|
|
public sealed class DriverHostActorWriteAskTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-write-ask-test");
|
|
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
|
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
|
|
private const string RawNodeId = "ns=2;s=Plant/Modbus/dev1/speed";
|
|
|
|
/// <summary>The value the gated driver holds; every other value is written at once.</summary>
|
|
private const double HeldValue = 99.0;
|
|
|
|
/// <summary>
|
|
/// A write still inside a 9 s write timeout is not cut off at 8 s: the host waits for the child, which
|
|
/// answers with the device's real outcome. (Mutation-checked: a fixed 8 s Ask on the same clock fails this
|
|
/// with "write timeout" at the 8.5 s step.)
|
|
/// </summary>
|
|
[Fact]
|
|
public void A_write_within_a_9s_write_timeout_is_not_failed_by_the_host_Ask()
|
|
{
|
|
var clock = new ManualTimeProvider();
|
|
var driver = new GatedWritableDriver();
|
|
var host = SpawnHostAndApply(driver, new DriverWriteBounds(TimeSpan.FromSeconds(9)), clock);
|
|
|
|
var asker = RouteHeldWrite(host, driver, clock);
|
|
clock.Advance(TimeSpan.FromSeconds(8.5));
|
|
asker.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
|
|
|
driver.Release();
|
|
var result = asker.ExpectMsg<DriverHostActor.NodeWriteResult>(Timeout);
|
|
result.Success.ShouldBeTrue(result.Reason);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A token-ignoring write under a 9 s write timeout is answered by the CHILD (abandoned BadTimeout at
|
|
/// 9 s + its grace), not by the host's Ask: the Ask covers the child's whole bound. Under the fixed 8 s Ask
|
|
/// this read "write timeout" from the Ask at 8 s while the device still had the write.
|
|
/// </summary>
|
|
[Fact]
|
|
public void A_token_ignoring_write_is_answered_by_the_child_before_the_host_Ask()
|
|
{
|
|
var clock = new ManualTimeProvider();
|
|
var driver = new GatedWritableDriver { IgnoreCancellation = true };
|
|
var bounds = new DriverWriteBounds(TimeSpan.FromSeconds(9));
|
|
var host = SpawnHostAndApply(driver, bounds, clock);
|
|
|
|
var asker = RouteHeldWrite(host, driver, clock);
|
|
clock.Advance(bounds.Timeout);
|
|
asker.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
|
clock.Advance(bounds.AbandonGrace);
|
|
|
|
var result = asker.ExpectMsg<DriverHostActor.NodeWriteResult>(Timeout);
|
|
result.Success.ShouldBeFalse();
|
|
result.Reason!.ShouldContain("800A0000", customMessage: "the child's BadTimeout, not the host Ask's \"write timeout\"");
|
|
result.Reason!.ShouldContain("within 9000 ms");
|
|
bounds.RunBound.ShouldBeLessThan(bounds.HostAsk);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The mechanism the host's Ask relies on: with an Infinite Akka timeout, the Ask's deadline is the write
|
|
/// clock's token alone — it does not complete a tick before <see cref="DriverWriteBounds.HostAsk"/>, and it
|
|
/// does complete (cancelled, which the host reports as "write timeout") at it, for a child that never answers.
|
|
/// </summary>
|
|
[Fact]
|
|
public void An_Ask_with_an_Infinite_timeout_ends_exactly_at_its_clock_deadline()
|
|
{
|
|
var clock = new ManualTimeProvider();
|
|
var bounds = new DriverWriteBounds(TimeSpan.FromSeconds(9));
|
|
var silentChild = CreateTestProbe();
|
|
|
|
using var deadline = new CancellationTokenSource(bounds.HostAsk, clock);
|
|
var ask = silentChild.Ref.Ask<DriverInstanceActor.WriteAttributeResult>(
|
|
new DriverInstanceActor.WriteAttribute("x", 1), System.Threading.Timeout.InfiniteTimeSpan, deadline.Token);
|
|
|
|
clock.Advance(bounds.HostAsk - TimeSpan.FromMilliseconds(1));
|
|
Thread.Sleep(200);
|
|
ask.IsCompleted.ShouldBeFalse("the Ask ended before its clock deadline");
|
|
|
|
clock.Advance(TimeSpan.FromMilliseconds(1));
|
|
AwaitCondition(() => ask.IsCompleted, Timeout, TimeSpan.FromMilliseconds(20));
|
|
ask.IsCompletedSuccessfully.ShouldBeFalse();
|
|
}
|
|
|
|
/// <summary>Routes the held write once the host accepts writes (a warm-up write proves the child is Connected
|
|
/// and the reverse map is pushed), then waits until it is parked in the driver with every bound armed.</summary>
|
|
private Akka.TestKit.TestProbe RouteHeldWrite(IActorRef host, GatedWritableDriver driver, ManualTimeProvider clock)
|
|
{
|
|
var lastReason = "(no reply)";
|
|
AwaitAssert(
|
|
() =>
|
|
{
|
|
var warm = CreateTestProbe();
|
|
host.Tell(new DriverHostActor.RouteNodeWrite(RawNodeId, 1.0, AddressSpaceRealm.Raw), warm.Ref);
|
|
var r = warm.ExpectMsg<DriverHostActor.NodeWriteResult>(TimeSpan.FromSeconds(1));
|
|
lastReason = r.Reason ?? "(none)";
|
|
r.Success.ShouldBeTrue($"host kept rejecting the warm-up write; last reason: {lastReason}");
|
|
},
|
|
duration: Timeout,
|
|
interval: TimeSpan.FromMilliseconds(50));
|
|
|
|
var asker = CreateTestProbe();
|
|
host.Tell(new DriverHostActor.RouteNodeWrite(RawNodeId, HeldValue, AddressSpaceRealm.Raw), asker.Ref);
|
|
AwaitCondition(() => driver.HeldEntered, Timeout, TimeSpan.FromMilliseconds(20));
|
|
// The host's Ask deadline + the child's token cancel + the child's abandon bound.
|
|
AwaitCondition(() => clock.ArmedTimers == 3, Timeout, TimeSpan.FromMilliseconds(20));
|
|
return asker;
|
|
}
|
|
|
|
private IActorRef SpawnHostAndApply(GatedWritableDriver driver, DriverWriteBounds bounds, ManualTimeProvider clock)
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDeployment(db);
|
|
var coordinator = CreateTestProbe();
|
|
var host = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
driverFactory: new SingleDriverFactory(driver),
|
|
localRoles: new HashSet<string> { "driver" },
|
|
writeBounds: bounds,
|
|
writeClock: clock));
|
|
|
|
host.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
return host;
|
|
}
|
|
|
|
/// <summary>One enabled Modbus driver "drv-1" with one ReadWrite tag, RawPath <c>Plant/Modbus/dev1/speed</c>.</summary>
|
|
private static DeploymentId SeedDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db)
|
|
{
|
|
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
|
DriverInstances = new[]
|
|
{
|
|
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
|
|
},
|
|
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = new[]
|
|
{
|
|
new { TagId = "t-speed", DeviceId = "dev-1", TagGroupId = (string?)null, Name = "speed", DataType = "Double", AccessLevel = 1, TagConfig = "{}" },
|
|
},
|
|
UnsAreas = Array.Empty<object>(),
|
|
UnsLines = Array.Empty<object>(),
|
|
Equipment = Array.Empty<object>(),
|
|
UnsTagReferences = Array.Empty<object>(),
|
|
});
|
|
|
|
var id = DeploymentId.NewId();
|
|
using var ctx = db.CreateDbContext();
|
|
ctx.Deployments.Add(new Deployment
|
|
{
|
|
DeploymentId = id.Value,
|
|
RevisionHash = RevA.Value,
|
|
Status = DeploymentStatus.Sealed,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = DateTime.UtcNow,
|
|
ArtifactBlob = artifact,
|
|
});
|
|
ctx.SaveChanges();
|
|
return id;
|
|
}
|
|
|
|
private sealed class SingleDriverFactory(GatedWritableDriver driver) : IDriverFactory
|
|
{
|
|
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) =>
|
|
driverType == "Modbus" ? driver.Bind(driverInstanceId, driverType) : null;
|
|
|
|
public IReadOnlyCollection<string> SupportedTypes => new[] { "Modbus" };
|
|
}
|
|
|
|
/// <summary>Writes <see cref="HeldValue"/> only once released (honouring its token unless
|
|
/// <see cref="IgnoreCancellation"/>); every other value is written at once.</summary>
|
|
private sealed class GatedWritableDriver : IDriver, IWritable
|
|
{
|
|
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private int _heldEntered;
|
|
|
|
public bool IgnoreCancellation { get; init; }
|
|
public bool HeldEntered => Volatile.Read(ref _heldEntered) == 1;
|
|
public string DriverInstanceId { get; private set; } = string.Empty;
|
|
public string DriverType { get; private set; } = string.Empty;
|
|
|
|
public GatedWritableDriver Bind(string id, string type)
|
|
{
|
|
DriverInstanceId = id;
|
|
DriverType = type;
|
|
return this;
|
|
}
|
|
|
|
public void Release() => _release.TrySetResult();
|
|
|
|
public async Task<IReadOnlyList<WriteResult>> WriteAsync(IReadOnlyList<WriteRequest> writes, CancellationToken cancellationToken)
|
|
{
|
|
if (writes.Any(w => w.Value is HeldValue))
|
|
{
|
|
Volatile.Write(ref _heldEntered, 1);
|
|
if (IgnoreCancellation) await _release.Task;
|
|
else await _release.Task.WaitAsync(cancellationToken);
|
|
}
|
|
|
|
return writes.Select(_ => new WriteResult(OpcUaStatus.Good)).ToList();
|
|
}
|
|
|
|
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
public Task ShutdownAsync(CancellationToken cancellationToken)
|
|
{
|
|
_release.TrySetResult();
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
|
}
|
|
}
|