Merge #868 fourth review fixes: poison only by SQL error allowlist, memory-counted acks survive restart, Stale-recovered revisions unconfirmed
Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
+91
@@ -495,6 +495,97 @@ public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTes
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fourth review M1: poison is an ALLOWLIST of SQL Server rejections. Microsoft.Data.SqlClient never sets
|
||||
/// <c>DbException.IsTransient</c>, so the old deny-list called every SQL Server transient poison.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(2628, true)]
|
||||
[InlineData(8152, true)]
|
||||
[InlineData(547, true)]
|
||||
[InlineData(515, true)]
|
||||
[InlineData(2627, true)]
|
||||
[InlineData(2601, true)]
|
||||
[InlineData(1205, false)] // deadlock victim
|
||||
[InlineData(40613, false)] // database unavailable (failover)
|
||||
[InlineData(10054, false)] // connection forcibly closed
|
||||
[InlineData(4060, false)] // cannot open database
|
||||
public void Only_known_SQL_rejections_are_poison(int number, bool poison)
|
||||
{
|
||||
var ex = new DbUpdateException("An error occurred while saving the entity changes.",
|
||||
SqlExceptionFactory.Create(number, $"error {number}"));
|
||||
ConfigPublishCoordinator.IsPoison(ex).ShouldBe(poison);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_timeout_a_concurrency_conflict_and_a_providerless_update_error_are_not_poison()
|
||||
{
|
||||
ConfigPublishCoordinator.IsPoison(new DbUpdateException("save failed", SqlExceptionFactory.Timeout())).ShouldBeFalse();
|
||||
ConfigPublishCoordinator.IsPoison(new DbUpdateConcurrencyException("conflict")).ShouldBeFalse();
|
||||
ConfigPublishCoordinator.IsPoison(new DbUpdateException("no provider detail")).ShouldBeFalse();
|
||||
ConfigPublishCoordinator.IsPoison(SqlExceptionFactory.Create(2628, "not wrapped by EF")).ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fourth review M1: a deadlock on the ack write is retried and the row is RECORDED once it clears — it used to
|
||||
/// be classed poison, counted from memory, and the node's row left Applying for good.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_deadlocked_ack_write_is_retried_and_recorded_not_counted_from_memory()
|
||||
{
|
||||
var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(NoTimeout);
|
||||
db.FailNodeStateUpdatesTransiently = true;
|
||||
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
Thread.Sleep(TimeSpan.FromMilliseconds(400));
|
||||
db.FailNodeStateUpdatesTransiently = false;
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
ctx.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fourth review M1: an ack counted only from memory (its write was rejected) must survive a coordinator restart
|
||||
/// before the outcome is written — no row records it, and its node will not resend.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_memory_counted_ack_survives_a_coordinator_restart()
|
||||
{
|
||||
var db = new ConstraintEnforcingDbFactory(Guid.NewGuid().ToString("N"));
|
||||
SeedNodes(db, Node, "site-a-2:4053");
|
||||
var deploymentId = SeedDeployment(db, DeploymentStatus.Dispatching, DateTime.UtcNow);
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
db, NoTimeout, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
// Node 1's write is rejected: counted from memory only.
|
||||
db.PoisonNodeStateUpdates = true;
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
Thread.Sleep(TimeSpan.FromMilliseconds(300));
|
||||
db.PoisonNodeStateUpdates = false;
|
||||
|
||||
// Restart the coordinator: a dispatch whose guard read cannot open the DB throws in the handler.
|
||||
db.FailNextContexts(1);
|
||||
coordinator.Tell(new DispatchDeployment(DeploymentId.NewId(), TestRevision, CorrelationId.NewId()));
|
||||
Thread.Sleep(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-2:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Single(d => d.DeploymentId == deploymentId.Value).Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
private (ConstraintEnforcingDbFactory Db, DeploymentId Id, IActorRef Coordinator) DispatchedOnConstraintDb(TimeSpan applyDeadline)
|
||||
{
|
||||
var db = new ConstraintEnforcingDbFactory(Guid.NewGuid().ToString("N"));
|
||||
|
||||
+23
-7
@@ -8,9 +8,10 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||
/// <summary>
|
||||
/// An in-memory ConfigDb that behaves like SQL Server where the InMemory provider does not (lmxopcua#868 third
|
||||
/// review): a <c>FailureReason</c> longer than the column refuses the whole <c>SaveChanges</c> with a
|
||||
/// <see cref="DbUpdateException"/>. It can also reject (<see cref="PoisonNodeStateUpdates"/>, a non-transient
|
||||
/// <see cref="DbUpdateException"/>) or fail transiently (<see cref="FailNodeStateUpdatesTransiently"/>) every
|
||||
/// update of an existing <c>NodeDeploymentState</c> row.
|
||||
/// <see cref="DbUpdateException"/> over SQL error 2628. It can also reject (<see cref="PoisonNodeStateUpdates"/>,
|
||||
/// error 547) or fail transiently (<see cref="FailNodeStateUpdatesTransiently"/>, deadlock 1205) every update of
|
||||
/// an existing <c>NodeDeploymentState</c> row — each a real <c>SqlException</c> inside EF's
|
||||
/// <see cref="DbUpdateException"/>, the shape SQL Server produces (fourth review M1).
|
||||
/// </summary>
|
||||
public sealed class ConstraintEnforcingDbFactory(string dbName) : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
@@ -22,8 +23,18 @@ public sealed class ConstraintEnforcingDbFactory(string dbName) : IDbContextFact
|
||||
/// <summary>Every update of an existing node row fails as if the connection dropped.</summary>
|
||||
public bool FailNodeStateUpdatesTransiently { get => _interceptor.Transient; set => _interceptor.Transient = value; }
|
||||
|
||||
private int _failNextContexts;
|
||||
|
||||
/// <summary>The next <paramref name="count"/> context creations throw (an unreachable server).</summary>
|
||||
public void FailNextContexts(int count) => Volatile.Write(ref _failNextContexts, count);
|
||||
|
||||
/// <inheritdoc />
|
||||
public OtOpcUaConfigDbContext CreateDbContext() => new(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
public OtOpcUaConfigDbContext CreateDbContext() =>
|
||||
Volatile.Read(ref _failNextContexts) > 0 && Interlocked.Decrement(ref _failNextContexts) >= 0
|
||||
? throw new InvalidOperationException("ConfigDb unreachable (simulated)")
|
||||
: Create();
|
||||
|
||||
private OtOpcUaConfigDbContext Create() => new(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
.UseInMemoryDatabase(dbName)
|
||||
.AddInterceptors(_interceptor)
|
||||
.Options);
|
||||
@@ -57,11 +68,16 @@ public sealed class ConstraintEnforcingDbFactory(string dbName) : IDbContextFact
|
||||
_ => null,
|
||||
};
|
||||
if (reason is { Length: > FailureReasons.MaxLength })
|
||||
throw new DbUpdateException("String or binary data would be truncated in column 'FailureReason'.");
|
||||
throw new DbUpdateException("An error occurred while saving the entity changes.",
|
||||
SqlExceptionFactory.Create(2628, "String or binary data would be truncated in column 'FailureReason'."));
|
||||
if (e.Entity is NodeDeploymentState && e.State == EntityState.Modified)
|
||||
{
|
||||
if (Poison) throw new DbUpdateException("The UPDATE statement conflicted with a CHECK constraint.");
|
||||
if (Transient) throw new InvalidOperationException("A network-related error occurred (simulated).");
|
||||
if (Poison)
|
||||
throw new DbUpdateException("An error occurred while saving the entity changes.",
|
||||
SqlExceptionFactory.Create(547, "The UPDATE statement conflicted with a CHECK constraint."));
|
||||
if (Transient)
|
||||
throw new DbUpdateException("An error occurred while saving the entity changes.",
|
||||
SqlExceptionFactory.Create(1205, "Transaction was deadlocked on lock resources and has been chosen as the deadlock victim."));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
using System.ComponentModel;
|
||||
using System.Reflection;
|
||||
using Microsoft.Data.SqlClient;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Builds real <see cref="SqlException"/>s for tests. The type has no public constructor, so this goes through its
|
||||
/// internal factory by reflection — the same objects SQL Server failures produce, carrying a real error number.
|
||||
/// </summary>
|
||||
public static class SqlExceptionFactory
|
||||
{
|
||||
private const BindingFlags NonPublic = BindingFlags.NonPublic | BindingFlags.Instance;
|
||||
|
||||
/// <summary>A <see cref="SqlException"/> with one error of the given number.</summary>
|
||||
/// <param name="number">The SQL Server error number (e.g. 2628, 1205, -2).</param>
|
||||
/// <param name="message">The error text.</param>
|
||||
/// <param name="inner">An inner exception (e.g. a <see cref="Win32Exception"/> for a timeout).</param>
|
||||
/// <returns>The exception.</returns>
|
||||
public static SqlException Create(int number, string message, Exception? inner = null)
|
||||
{
|
||||
var errorCtor = typeof(SqlError).GetConstructors(NonPublic)
|
||||
.OrderByDescending(c => c.GetParameters().Length)
|
||||
.First();
|
||||
var args = errorCtor.GetParameters().Select(p => p.Name switch
|
||||
{
|
||||
"infoNumber" => (object?)number,
|
||||
"errorMessage" => message,
|
||||
"server" => "test-sql",
|
||||
"procedure" => string.Empty,
|
||||
"exception" => inner,
|
||||
"errorClass" => (byte)16,
|
||||
_ => p.ParameterType == typeof(string) ? string.Empty
|
||||
: p.ParameterType.IsValueType ? Activator.CreateInstance(p.ParameterType) : null,
|
||||
}).ToArray();
|
||||
var error = (SqlError)errorCtor.Invoke(args);
|
||||
|
||||
var collection = (SqlErrorCollection)Activator.CreateInstance(typeof(SqlErrorCollection), nonPublic: true)!;
|
||||
typeof(SqlErrorCollection).GetMethod("Add", NonPublic)!.Invoke(collection, [error]);
|
||||
|
||||
var create = typeof(SqlException)
|
||||
.GetMethods(BindingFlags.NonPublic | BindingFlags.Static)
|
||||
.Where(m => m.Name == "CreateException")
|
||||
.Select(m => (Method: m, Params: m.GetParameters()))
|
||||
.Where(x => x.Params.Length >= 2 && x.Params[0].ParameterType == typeof(SqlErrorCollection)
|
||||
&& x.Params[1].ParameterType == typeof(string))
|
||||
.OrderBy(x => x.Params.Length)
|
||||
.First();
|
||||
var createArgs = create.Params.Select((p, i) => i switch
|
||||
{
|
||||
0 => collection,
|
||||
1 => (object?)"16.0.0",
|
||||
_ => p.ParameterType == typeof(Exception) ? inner
|
||||
: p.ParameterType.IsValueType ? Activator.CreateInstance(p.ParameterType) : null,
|
||||
}).ToArray();
|
||||
return (SqlException)create.Method.Invoke(null, createArgs)!;
|
||||
}
|
||||
|
||||
/// <summary>A command timeout as SQL Server reports it: number -2 over a Win32 258 (WAIT_TIMEOUT).</summary>
|
||||
/// <returns>The exception.</returns>
|
||||
public static SqlException Timeout() =>
|
||||
Create(-2, "Execution Timeout Expired.", new Win32Exception(258, "The wait operation timed out."));
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#868 fourth review M2: a revision a node ADOPTS without an outcome-reporting apply — Stale recovery, a
|
||||
/// failed bootstrap restore — is not known to be served. A dispatch of that same revision must RE-APPLY (the
|
||||
/// publish actor gets a rebuild) instead of short-circuiting a clean Applied over an empty address space.
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorUnconfirmedRevisionTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
|
||||
[Fact]
|
||||
public void A_revision_adopted_by_Stale_recovery_is_reapplied_on_redispatch()
|
||||
{
|
||||
var db = new SwitchableDbFactory(Guid.NewGuid().ToString("N"));
|
||||
var deploymentId = Seed(db);
|
||||
db.Down = true;
|
||||
var publish = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, CreateTestProbe().Ref, localRoles: new HashSet<string> { "driver" },
|
||||
opcUaPublishActor: publish.Ref));
|
||||
|
||||
// Bootstrap fails (DB down) ⇒ Stale. Bring the DB back and drive the recovery tick by hand.
|
||||
// Barrier: Stale answers RouteNodeWrite with a "stale" refusal, so a reply proves the host is in Stale.
|
||||
var asker = CreateTestProbe();
|
||||
actor.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, ZB.MOM.WW.OtOpcUa.Commons.OpcUa.AddressSpaceRealm.Uns), asker.Ref);
|
||||
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget, cancellationToken: TestContext.Current.CancellationToken)
|
||||
.Reason!.ShouldContain("stale");
|
||||
db.Down = false;
|
||||
actor.Tell(DriverHostActor.RetryConfigDbConnection.Instance);
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value).Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
}, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken); // recovery restores nothing
|
||||
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
|
||||
publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(
|
||||
m => m.DeploymentId == deploymentId && m.ReportOutcome, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_revision_whose_bootstrap_restore_failed_is_reapplied_on_redispatch()
|
||||
{
|
||||
var db = new SwitchableDbFactory(Guid.NewGuid().ToString("N"));
|
||||
var deploymentId = Seed(db);
|
||||
using (var ctx = db.CreateDbContext())
|
||||
{
|
||||
ctx.NodeDeploymentStates.Add(new NodeDeploymentState
|
||||
{
|
||||
NodeId = TestNode.Value, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied,
|
||||
StartedAtUtc = DateTime.UtcNow, AppliedAtUtc = DateTime.UtcNow,
|
||||
});
|
||||
ctx.SaveChanges();
|
||||
}
|
||||
db.FailRestore = true; // the restore's artifact read throws; the bootstrap's own row read does not
|
||||
var publish = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, CreateTestProbe().Ref, localRoles: new HashSet<string> { "driver" },
|
||||
opcUaPublishActor: publish.Ref));
|
||||
AwaitCondition(() => db.RestoreFailures > 0, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
db.FailRestore = false;
|
||||
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
|
||||
publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(
|
||||
m => m.DeploymentId == deploymentId && m.ReportOutcome, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
private static DeploymentId Seed(IDbContextFactory<OtOpcUaConfigDbContext> db) =>
|
||||
V3ArtifactSeeder.SeedV3Deployment(db, RevA,
|
||||
rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) },
|
||||
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
|
||||
|
||||
/// <summary>In-memory ConfigDb that can be down, or fail only the bootstrap restore's artifact read.</summary>
|
||||
private sealed class SwitchableDbFactory(string dbName) : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
private volatile bool _down;
|
||||
private volatile bool _failRestore;
|
||||
private int _attempts;
|
||||
private int _restoreFailures;
|
||||
|
||||
public bool Down { get => _down; set => _down = value; }
|
||||
public bool FailRestore { get => _failRestore; set => _failRestore = value; }
|
||||
public int Attempts => Volatile.Read(ref _attempts);
|
||||
public int RestoreFailures => Volatile.Read(ref _restoreFailures);
|
||||
|
||||
public OtOpcUaConfigDbContext CreateDbContext()
|
||||
{
|
||||
Interlocked.Increment(ref _attempts);
|
||||
if (_down) throw new InvalidOperationException("ConfigDb unreachable (simulated)");
|
||||
if (_failRestore && new System.Diagnostics.StackTrace().ToString().Contains("RestoreApplied"))
|
||||
{
|
||||
Interlocked.Increment(ref _restoreFailures);
|
||||
throw new InvalidOperationException("ConfigDb read failed during restore (simulated)");
|
||||
}
|
||||
return new OtOpcUaConfigDbContext(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
.UseInMemoryDatabase(dbName).Options);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user