ec3b835e4a
M1 IsPoison classified every SQL Server transient as poison: Microsoft.Data.SqlClient never sets DbException.IsTransient, so a deadlock (1205), timeout (-2 / Win32 258), dropped connection or AG failover was counted from memory and the node's row left Applying. Poison is now an ALLOWLIST: a DbUpdateException (not concurrency) carrying a SqlException whose number is 8152, 2628, 547, 515, 2627 or 2601. Everything else retries, bounded by the deadline deferral cap. IsPoison is public for tests. Acks counted only from memory (poison branch and the deferral cap) are kept in _memoryOnlyAcks and re-sent by PreRestart alongside _unpersisted, so a restart before the terminal write does not lose them. M2 Revisions adopted without an outcome-reporting apply are unconfirmed, so a same-revision dispatch re-applies instead of short-circuiting a clean Applied over an empty address space: TryRecoverFromStale (restores only the revision marker) and RestoreApplied when the artifact read fails (null blob, LoadArtifactBlob swallows the error) or the restore throws. L1 The re-drive comment documents the CreatedAtUtc recompute after a second restart / failover. L2 _currentRevisionDegraded renamed _currentRevisionUnconfirmed; the stale TryBootFromCache remark corrected. L3 docs/Redundancy.md: a Direct node running from cache that still cannot reach the ConfigDb acks a same-revision re-dispatch Failed (PartiallyFailed, not Sealed); Stale recovery and failed restores are unconfirmed too. Test harness: SqlExceptionFactory builds real SqlExceptions (reflection on SqlError / SqlErrorCollection / SqlException.CreateException); ConstraintEnforcingDbFactory now throws DbUpdateException over SQL 2628 (over-length), 547 (poison) and 1205 (transient), plus FailNextContexts. Tests (all fail on the previous code): poison allowlist theory (1205, 40613, 10054, 4060 not poison) + timeout/concurrency/providerless not poison; a deadlocked ack write is retried and recorded; a memory-counted ack survives a coordinator restart; Stale-recovered and failed-restore revisions re-apply on re-dispatch. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
118 lines
6.0 KiB
C#
118 lines
6.0 KiB
C#
using Akka.Actor;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// lmxopcua#868 fourth review M2: a revision a node ADOPTS without an outcome-reporting apply — Stale recovery, a
|
|
/// failed bootstrap restore — is not known to be served. A dispatch of that same revision must RE-APPLY (the
|
|
/// publish actor gets a rebuild) instead of short-circuiting a clean Applied over an empty address space.
|
|
/// </summary>
|
|
public sealed class DriverHostActorUnconfirmedRevisionTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
|
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
|
|
|
[Fact]
|
|
public void A_revision_adopted_by_Stale_recovery_is_reapplied_on_redispatch()
|
|
{
|
|
var db = new SwitchableDbFactory(Guid.NewGuid().ToString("N"));
|
|
var deploymentId = Seed(db);
|
|
db.Down = true;
|
|
var publish = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, CreateTestProbe().Ref, localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish.Ref));
|
|
|
|
// Bootstrap fails (DB down) ⇒ Stale. Bring the DB back and drive the recovery tick by hand.
|
|
// Barrier: Stale answers RouteNodeWrite with a "stale" refusal, so a reply proves the host is in Stale.
|
|
var asker = CreateTestProbe();
|
|
actor.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, ZB.MOM.WW.OtOpcUa.Commons.OpcUa.AddressSpaceRealm.Uns), asker.Ref);
|
|
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget, cancellationToken: TestContext.Current.CancellationToken)
|
|
.Reason!.ShouldContain("stale");
|
|
db.Down = false;
|
|
actor.Tell(DriverHostActor.RetryConfigDbConnection.Instance);
|
|
AwaitAssert(() =>
|
|
{
|
|
using var ctx = db.CreateDbContext();
|
|
ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value).Status.ShouldBe(NodeDeploymentStatus.Applied);
|
|
}, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken); // recovery restores nothing
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(
|
|
m => m.DeploymentId == deploymentId && m.ReportOutcome, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
}
|
|
|
|
[Fact]
|
|
public void A_revision_whose_bootstrap_restore_failed_is_reapplied_on_redispatch()
|
|
{
|
|
var db = new SwitchableDbFactory(Guid.NewGuid().ToString("N"));
|
|
var deploymentId = Seed(db);
|
|
using (var ctx = db.CreateDbContext())
|
|
{
|
|
ctx.NodeDeploymentStates.Add(new NodeDeploymentState
|
|
{
|
|
NodeId = TestNode.Value, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied,
|
|
StartedAtUtc = DateTime.UtcNow, AppliedAtUtc = DateTime.UtcNow,
|
|
});
|
|
ctx.SaveChanges();
|
|
}
|
|
db.FailRestore = true; // the restore's artifact read throws; the bootstrap's own row read does not
|
|
var publish = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, CreateTestProbe().Ref, localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish.Ref));
|
|
AwaitCondition(() => db.RestoreFailures > 0, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
db.FailRestore = false;
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(
|
|
m => m.DeploymentId == deploymentId && m.ReportOutcome, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
}
|
|
|
|
private static DeploymentId Seed(IDbContextFactory<OtOpcUaConfigDbContext> db) =>
|
|
V3ArtifactSeeder.SeedV3Deployment(db, RevA,
|
|
rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) },
|
|
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
|
|
|
|
/// <summary>In-memory ConfigDb that can be down, or fail only the bootstrap restore's artifact read.</summary>
|
|
private sealed class SwitchableDbFactory(string dbName) : IDbContextFactory<OtOpcUaConfigDbContext>
|
|
{
|
|
private volatile bool _down;
|
|
private volatile bool _failRestore;
|
|
private int _attempts;
|
|
private int _restoreFailures;
|
|
|
|
public bool Down { get => _down; set => _down = value; }
|
|
public bool FailRestore { get => _failRestore; set => _failRestore = value; }
|
|
public int Attempts => Volatile.Read(ref _attempts);
|
|
public int RestoreFailures => Volatile.Read(ref _restoreFailures);
|
|
|
|
public OtOpcUaConfigDbContext CreateDbContext()
|
|
{
|
|
Interlocked.Increment(ref _attempts);
|
|
if (_down) throw new InvalidOperationException("ConfigDb unreachable (simulated)");
|
|
if (_failRestore && new System.Diagnostics.StackTrace().ToString().Contains("RestoreApplied"))
|
|
{
|
|
Interlocked.Increment(ref _restoreFailures);
|
|
throw new InvalidOperationException("ConfigDb read failed during restore (simulated)");
|
|
}
|
|
return new OtOpcUaConfigDbContext(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
|
.UseInMemoryDatabase(dbName).Options);
|
|
}
|
|
}
|
|
}
|