0b7c53f64f
Comparing full-suite runs on this branch against master in a clean worktree: both fail 11 tests, 9 identical. Master's two extras are load-flaky unit tests (AbCip Probe_loops, Galaxy EventPumpBoundedChannel); this branch's two extras were both Host.IntegrationTests deploy-path tests — the area this change re-times — so they are attributable here rather than to background flakiness. Cause is margin, not correctness. These waits used to observe a coordinator that sealed instantly on an empty expected-ack set; they now observe a real ApplyAck round-trip from every configured node. 15s was enormous margin against "instant" and thin against the real thing, so they failed only under full-suite CPU contention. Hoisted to TwoNodeClusterHarness.DeploySealTimeout (45s) so the reason is recorded once rather than as four unexplained numbers. DriverReconnectE2eTests is deliberately left alone: it seeds both ClusterNode rows itself and unconditionally, so its expected-ack set is identical before and after this change. Widening its timeout would be papering over a flake this change did not cause. Host.IntegrationTests 195/201; sole failure AbCip_Green_AgainstSim, verified failing on master. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
120 lines
5.2 KiB
C#
120 lines
5.2 KiB
C#
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
|
|
|
/// <summary>
|
|
/// End-to-end deploy: AdminOperationsActor → ConfigPublishCoordinator → DriverHostActor on both
|
|
/// nodes → ApplyAck → coordinator seals. Verifies the cluster singleton + DistributedPubSub
|
|
/// dispatch path actually works against the real Akka cluster the harness builds.
|
|
/// </summary>
|
|
public sealed class DeployHappyPathTests
|
|
{
|
|
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
|
|
|
/// <summary>Verifies that StartDeployment seals after both nodes apply.</summary>
|
|
[Fact]
|
|
public async Task StartDeployment_seals_after_both_nodes_apply()
|
|
{
|
|
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
|
await harness.SeedDefaultClusterAsync();
|
|
|
|
await using var scope = harness.NodeA.Services.CreateAsyncScope();
|
|
var client = scope.ServiceProvider.GetRequiredService<IAdminOperationsClient>();
|
|
|
|
var result = await client.StartDeploymentAsync(createdBy: "alice@test", Ct);
|
|
|
|
result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
|
|
result.DeploymentId.ShouldNotBeNull();
|
|
var deploymentId = result.DeploymentId!.Value.Value;
|
|
|
|
// Wait up to 15s for coordinator to seal after both DriverHostActors ack.
|
|
await WaitForAsync(async () =>
|
|
{
|
|
await using var db = await CreateDbAsync(harness);
|
|
var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(d => d.DeploymentId == deploymentId, Ct);
|
|
return d?.Status == DeploymentStatus.Sealed;
|
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
|
|
|
await using var db = await CreateDbAsync(harness);
|
|
var deployment = await db.Deployments.AsNoTracking()
|
|
.FirstAsync(d => d.DeploymentId == deploymentId, Ct);
|
|
deployment.Status.ShouldBe(DeploymentStatus.Sealed);
|
|
deployment.SealedAtUtc.ShouldNotBeNull();
|
|
|
|
var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
|
|
.Where(s => s.DeploymentId == deploymentId)
|
|
.ToListAsync(Ct);
|
|
nodeStates.Count.ShouldBe(2);
|
|
nodeStates.ShouldAllBe(s => s.Status == NodeDeploymentStatus.Applied);
|
|
}
|
|
|
|
/// <summary>Verifies that replaying dispatch to same revision is idempotent and a no-op.</summary>
|
|
[Fact]
|
|
public async Task Replaying_dispatch_to_same_revision_is_idempotent_no_op()
|
|
{
|
|
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
|
await harness.SeedDefaultClusterAsync();
|
|
|
|
await using var scope = harness.NodeA.Services.CreateAsyncScope();
|
|
var client = scope.ServiceProvider.GetRequiredService<IAdminOperationsClient>();
|
|
|
|
var first = await client.StartDeploymentAsync(createdBy: "alice@test", Ct);
|
|
first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
|
|
|
|
await WaitForAsync(async () =>
|
|
{
|
|
await using var db = await CreateDbAsync(harness);
|
|
var d = await db.Deployments.AsNoTracking()
|
|
.FirstOrDefaultAsync(d => d.DeploymentId == first.DeploymentId!.Value.Value, Ct);
|
|
return d?.Status == DeploymentStatus.Sealed;
|
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
|
|
|
// Same DB state → same revision hash → AdminOperations short-circuits with NoChanges,
|
|
// OR (if it accepts) the second dispatch is idempotent (DriverHostActor recognises
|
|
// _currentRevision == msg.RevisionHash and acks Applied without re-applying).
|
|
var second = await client.StartDeploymentAsync(createdBy: "alice@test", Ct);
|
|
|
|
if (second.Outcome == StartDeploymentOutcome.Accepted)
|
|
{
|
|
// Second deployment row should also seal quickly because both drivers immediate-ack.
|
|
await WaitForAsync(async () =>
|
|
{
|
|
await using var db = await CreateDbAsync(harness);
|
|
var d = await db.Deployments.AsNoTracking()
|
|
.FirstOrDefaultAsync(d => d.DeploymentId == second.DeploymentId!.Value.Value, Ct);
|
|
return d?.Status == DeploymentStatus.Sealed;
|
|
}, TwoNodeClusterHarness.DeploySealTimeout);
|
|
|
|
second.RevisionHash!.Value.Value.ShouldBe(first.RevisionHash!.Value.Value);
|
|
}
|
|
else
|
|
{
|
|
second.Outcome.ShouldBeOneOf(StartDeploymentOutcome.NoChanges, StartDeploymentOutcome.AnotherDeploymentInFlight);
|
|
}
|
|
}
|
|
|
|
private static async Task<OtOpcUaConfigDbContext> CreateDbAsync(TwoNodeClusterHarness harness)
|
|
{
|
|
var factory = harness.NodeA.Services.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>();
|
|
return await factory.CreateDbContextAsync();
|
|
}
|
|
|
|
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout)
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
if (await condition()) return;
|
|
await Task.Delay(200);
|
|
}
|
|
throw new TimeoutException($"Condition not met within {timeout}");
|
|
}
|
|
}
|