Joseph Doherty
513fa1aa16
Comment/XML-doc changes only: - add summaries to ~10k test methods, fixtures, fakes and helpers (+ <param> on theories, <returns> on async Task methods, <inheritdoc/> on interface implementations the checker can resolve) - strip ~2.6k project-tracking references (WP*, P*-*, row-NN, Task N, #NN, Component-0NN) from comments, keeping the explanatory text; reword false positives in comments only - real docs on fakes implementing BCL/library interfaces (IDisposable, ILogger, IOptionsMonitor, IAsyncLifetime) - reword the last 5 src/ false positives (CWE-1236, failures-1, year-0001, M365) Only non-comment edits: four one-line nested classes split across lines so members can carry docs. Build 0 errors / 0 warnings.
785 lines
36 KiB
C#
785 lines
36 KiB
C#
using System.Text.Json;
|
|
using Akka.Actor;
|
|
using Akka.TestKit.Xunit2;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
|
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
|
|
|
/// <summary>
|
|
/// Regression tests for deferred register rows <b>78</b> and <b>72</b> — the two ways the site
|
|
/// runtime silently diverges from its own durable <c>deployed_configurations</c> rows.
|
|
///
|
|
/// <para>
|
|
/// <b>Row 78 (availability).</b> Instance Actors were created from three paths: the boot-time
|
|
/// SQLite load, an explicit deploy/enable command, and (row 67) the startup reconcile's
|
|
/// <see cref="EnsureInstanceStartedCommand"/> for configs the reconcile ITSELF fetched. A config
|
|
/// that arrives by LocalDb CDC <i>replication</i> hits none of them: the sync applier writes the
|
|
/// row straight into the file, AFTER the boot load has read storage and invisibly to the
|
|
/// reconcile, which correctly reports no gap because central's view of the site is right. Proven
|
|
/// live 2026-08-15 — a fresh site-a-a was seeded from its peer within ~5 s of boot and its
|
|
/// enabled instances sat dead, with their configs present, until another restart.
|
|
/// </para>
|
|
///
|
|
/// <para>
|
|
/// <b>Row 72 (silent divergence).</b> The reconcile refreshes a STALE row via
|
|
/// <c>StoreDeployedConfigIfNewerAsync</c>, but the running Instance Actor keeps serving the OLD
|
|
/// compiled config until the next restart or redeploy, and nothing surfaced
|
|
/// "stored revision != running revision".
|
|
/// </para>
|
|
///
|
|
/// <para>
|
|
/// Both are closed by one periodic sweep owned by the Deployment Manager singleton. Design note:
|
|
/// <c>docs/plans/2026-08-15-instance-sweep-self-heal-design.md</c>.
|
|
/// </para>
|
|
/// </summary>
|
|
public class InstanceSweepSelfHealTests : TestKit, IDisposable
|
|
{
|
|
private readonly SiteStorageService _storage;
|
|
private readonly ScriptCompilationService _compilationService;
|
|
private readonly SharedScriptLibrary _sharedScriptLibrary;
|
|
private readonly TestLocalDb _localDb;
|
|
|
|
/// <summary>Creates a temporary LocalDb-backed site store and the compilation services for each test.</summary>
|
|
public InstanceSweepSelfHealTests()
|
|
{
|
|
_localDb = TestLocalDb.CreateTemp("instance-sweep-test");
|
|
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
|
_storage.InitializeAsync().GetAwaiter().GetResult();
|
|
_compilationService = new ScriptCompilationService(
|
|
NullLogger<ScriptCompilationService>.Instance);
|
|
_sharedScriptLibrary = new SharedScriptLibrary(
|
|
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
void IDisposable.Dispose()
|
|
{
|
|
// Shut the actors down BEFORE the ILocalDb they write through is disposed.
|
|
Shutdown();
|
|
var path = _localDb.Path;
|
|
_localDb.Dispose();
|
|
TestLocalDb.DeleteFiles(path);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Sweep interval used by these tests. One second is the shortest the (whole-second) option
|
|
/// allows; every assertion below therefore budgets several seconds so it is waiting on the
|
|
/// behaviour, not on the clock.
|
|
/// </summary>
|
|
private const int SweepSeconds = 1;
|
|
|
|
/// <summary>Long enough for at least three sweep ticks to have come and gone.</summary>
|
|
private static readonly TimeSpan SeveralSweeps = TimeSpan.FromSeconds(4);
|
|
|
|
/// <summary>
|
|
/// Starts a Deployment Manager and blocks until its boot-time config load has PROVABLY read
|
|
/// storage (its first instance-count report). Load-bearing for every test here: a row written
|
|
/// before that read is picked up by the boot load itself, which is precisely the path row 78
|
|
/// is about NOT having. Without this gate a test would silently prove the boot load works.
|
|
/// </summary>
|
|
private IActorRef CreateBootedDeploymentManager(
|
|
FakeSiteEventLogger? siteLog = null,
|
|
int sweepSeconds = SweepSeconds)
|
|
{
|
|
var health = new DeploymentManagerStartupProbe();
|
|
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
|
|
_storage,
|
|
_compilationService,
|
|
_sharedScriptLibrary,
|
|
null,
|
|
new SiteRuntimeOptions { InstanceSweepIntervalSeconds = sweepSeconds },
|
|
NullLogger<DeploymentManagerActor>.Instance,
|
|
dclManager: null,
|
|
healthCollector: health,
|
|
serviceProvider: siteLog == null ? null : new SingleServiceProvider(siteLog),
|
|
loggerFactory: null,
|
|
configFetcher: null)));
|
|
|
|
AwaitCondition(() => health.ReportCount > 0, TimeSpan.FromSeconds(20),
|
|
TimeSpan.FromMilliseconds(25));
|
|
return dm;
|
|
}
|
|
|
|
private static string MakeConfigJson(string instanceName, string attributeValue = "42") =>
|
|
JsonSerializer.Serialize(new FlattenedConfiguration
|
|
{
|
|
InstanceUniqueName = instanceName,
|
|
Attributes =
|
|
[
|
|
new ResolvedAttribute
|
|
{
|
|
CanonicalName = "TestAttr", Value = attributeValue, DataType = "Int32"
|
|
}
|
|
]
|
|
});
|
|
|
|
/// <summary>
|
|
/// A config carrying a real script, so its compile gate / compile warm takes long enough for
|
|
/// the deploy pipeline to hold the instance across a sweep tick.
|
|
/// </summary>
|
|
private static string MakeScriptedConfigJson(string instanceName, string scriptCode) =>
|
|
JsonSerializer.Serialize(new FlattenedConfiguration
|
|
{
|
|
InstanceUniqueName = instanceName,
|
|
Attributes =
|
|
[
|
|
new ResolvedAttribute
|
|
{
|
|
CanonicalName = "TestAttr", Value = "42", DataType = "Int32"
|
|
}
|
|
],
|
|
Scripts =
|
|
[
|
|
new ResolvedScript
|
|
{
|
|
CanonicalName = "Worker", Code = scriptCode, TriggerType = "Call"
|
|
}
|
|
]
|
|
});
|
|
|
|
/// <summary>
|
|
/// Asks the Deployment Manager for a debug snapshot — the exact probe that answered
|
|
/// <c>instanceNotFound</c> on the live rig.
|
|
/// </summary>
|
|
private async Task<DebugViewSnapshot> SnapshotAsync(IActorRef dm, string instanceName) =>
|
|
await dm.Ask<DebugViewSnapshot>(
|
|
new SubscribeDebugViewRequest(instanceName, Guid.NewGuid().ToString()),
|
|
TimeSpan.FromSeconds(5));
|
|
|
|
private async Task<bool> InstanceIsRunningAsync(IActorRef dm, string instanceName) =>
|
|
!(await SnapshotAsync(dm, instanceName)).InstanceNotFound;
|
|
|
|
private async Task<IActorRef> ResolveInstanceChildAsync(IActorRef dm, string instanceName) =>
|
|
await Sys.ActorSelection(dm.Path / instanceName).ResolveOne(TimeSpan.FromSeconds(5));
|
|
|
|
/// <summary>
|
|
/// Site events announcing a sweep REPAIR (a start or a stale-revision restart). The terminal
|
|
/// "self-heal failed" event is deliberately excluded — it reports the outcome of a repair
|
|
/// already counted here, so counting both would double-count a single attempt.
|
|
/// </summary>
|
|
private static IReadOnlyList<FakeSiteEventLogger.Entry> SelfHealEvents(
|
|
FakeSiteEventLogger log, string instanceName) =>
|
|
log.OfType("deployment")
|
|
.Where(e => e.InstanceId == instanceName &&
|
|
e.Message.Contains("self-heal", StringComparison.OrdinalIgnoreCase) &&
|
|
!e.Message.Contains("self-heal failed", StringComparison.OrdinalIgnoreCase))
|
|
.ToArray();
|
|
|
|
/// <summary>
|
|
/// Deploys an instance the ordinary way and blocks until it is running, so a test can start
|
|
/// from a known-good runtime state.
|
|
/// </summary>
|
|
private async Task DeployAndAwaitAsync(
|
|
IActorRef dm, string instanceName, string revision, string configJson)
|
|
{
|
|
dm.Tell(new DeployInstanceCommand(
|
|
$"dep-{revision}", instanceName, revision, configJson, "admin", DateTimeOffset.UtcNow));
|
|
var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(15));
|
|
Assert.Equal(DeploymentStatus.Success, response.Status);
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
// ── Row 78: a config that arrives with no deploy, no enable and no reconcile ──
|
|
|
|
/// <summary>
|
|
/// THE reproduction of the missing-start bug. The node boots with an empty deployed-config store (so the
|
|
/// boot-time load starts nothing), and the config for an ENABLED instance then appears in
|
|
/// storage on its own — exactly what LocalDb's CDC sync applier does when it seeds a fresh
|
|
/// node from its peer. No <see cref="DeployInstanceCommand"/>, no
|
|
/// <see cref="EnsureInstanceStartedCommand"/>, no reconcile actor: nothing but the row.
|
|
///
|
|
/// <para>
|
|
/// <b>Fails before the fix</b> at the final assert — the row is in storage and the debug
|
|
/// snapshot still answers <c>InstanceNotFound = true</c>, the live symptom verbatim.
|
|
/// </para>
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Sweep_StartsEnabledInstance_WhoseConfigArrivedWithoutADeployOrReconcile()
|
|
{
|
|
const string instanceName = "ReplicatedPump";
|
|
|
|
var dm = CreateBootedDeploymentManager();
|
|
|
|
// Baseline: the node genuinely booted with nothing.
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.False(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(5));
|
|
|
|
// Replication seeds the row. This is a bare storage write on purpose — it is the only
|
|
// side effect the CDC applier has, and it is invisible to every other start path.
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-replicated", "rev-replicated", true);
|
|
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.True(
|
|
await InstanceIsRunningAsync(dm, instanceName),
|
|
"A replication-seeded config for an enabled instance left no running Instance "
|
|
+ "Actor — the instance would stay dead until the next restart (register row 78)"),
|
|
TimeSpan.FromSeconds(20));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The heal must be attributable: it emits the same Warning <c>deployment</c> site event
|
|
/// pattern as row 67, with the trigger named so an operator can tell a sweep heal from a
|
|
/// startup-reconcile heal.
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task SweepStartedInstance_EmitsAWarningSiteEvent_NamingTheSweepTrigger()
|
|
{
|
|
const string instanceName = "SurfacedSweepPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog);
|
|
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-s", "rev-s", true);
|
|
|
|
await AwaitAssertAsync(() =>
|
|
{
|
|
Assert.Contains(siteLog.OfType("deployment"), e =>
|
|
e.InstanceId == instanceName &&
|
|
e.Severity == "Warning" &&
|
|
e.Source == "DeploymentManagerActor" &&
|
|
e.Message.Contains("deployment-manager sweep", StringComparison.OrdinalIgnoreCase) &&
|
|
e.Message.Contains("self-heal", StringComparison.OrdinalIgnoreCase));
|
|
return Task.CompletedTask;
|
|
}, TimeSpan.FromSeconds(20));
|
|
}
|
|
|
|
// ── Row 78 negative guards ──
|
|
|
|
/// <summary>
|
|
/// A DISABLED instance legitimately has a config row and no running actor. The sweep must
|
|
/// never start it — that would silently undo an operator's disable.
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Sweep_NeverStarts_ADisabledInstance()
|
|
{
|
|
const string instanceName = "DisabledSweepPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog);
|
|
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-d", "rev-d", false);
|
|
|
|
await Task.Delay(SeveralSweeps);
|
|
|
|
Assert.False(await InstanceIsRunningAsync(dm, instanceName));
|
|
Assert.Empty(SelfHealEvents(siteLog, instanceName));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Idempotence: many sweeps, one start. The sweep runs forever on a live node, so a repair
|
|
/// that re-fired every tick would be a restart loop plus an unbounded stream of Warning site
|
|
/// events.
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Sweep_StartsTheInstanceExactlyOnce_AcrossManySweeps()
|
|
{
|
|
const string instanceName = "IdempotentSweepPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog);
|
|
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-i", "rev-i", true);
|
|
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(20));
|
|
|
|
var afterFirstHeal = await ResolveInstanceChildAsync(dm, instanceName);
|
|
|
|
// Let several more sweeps run over the now-healthy state.
|
|
await Task.Delay(SeveralSweeps);
|
|
|
|
Assert.Equal(afterFirstHeal, await ResolveInstanceChildAsync(dm, instanceName));
|
|
Assert.Single(SelfHealEvents(siteLog, instanceName));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A sweep that finds nothing wrong must be completely silent — no site events at all. This
|
|
/// runs on every tick of every healthy site, forever.
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Sweep_IsSilent_WhenThereIsNothingToHeal()
|
|
{
|
|
const string instanceName = "HealthySweepPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog);
|
|
|
|
await DeployAndAwaitAsync(dm, instanceName, "rev-1", MakeConfigJson(instanceName));
|
|
|
|
var eventsAfterDeploy = siteLog.Entries.Count;
|
|
await Task.Delay(SeveralSweeps);
|
|
|
|
Assert.Equal(eventsAfterDeploy, siteLog.Entries.Count);
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The sweep is opt-out: <c>InstanceSweepIntervalSeconds = 0</c> arms no timer at all, so a
|
|
/// site that wants the pre-fix behaviour keeps it (and the option's disable path is proven,
|
|
/// not merely validated).
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Sweep_DoesNothing_WhenTheIntervalIsZero()
|
|
{
|
|
const string instanceName = "NoSweepPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog, sweepSeconds: 0);
|
|
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-z", "rev-z", true);
|
|
|
|
await Task.Delay(SeveralSweeps);
|
|
|
|
Assert.False(await InstanceIsRunningAsync(dm, instanceName));
|
|
Assert.Empty(SelfHealEvents(siteLog, instanceName));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The sweep must defer to any deploy pipeline that owns the instance. A redeploy passes
|
|
/// through a window where the predecessor has been stopped and removed from routing but its
|
|
/// replacement does not exist yet — the sweep would read that as "enabled config, no actor"
|
|
/// and try to create a child whose actor name is still registered.
|
|
///
|
|
/// <para>
|
|
/// Sweep ticks are fired continuously across the whole redeploy so the window is genuinely
|
|
/// sampled; the assertion is that the redeploy succeeded untouched and the sweep announced
|
|
/// nothing.
|
|
/// </para>
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Sweep_DefersToAnInFlightRedeploy()
|
|
{
|
|
const string instanceName = "RedeployingPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog);
|
|
|
|
await DeployAndAwaitAsync(
|
|
dm, instanceName, "rev-1",
|
|
MakeScriptedConfigJson(instanceName, "var a = 1; var b = a + 1; return b;"));
|
|
|
|
using var ticking = new CancellationTokenSource();
|
|
var ticker = Task.Run(async () =>
|
|
{
|
|
while (!ticking.IsCancellationRequested)
|
|
{
|
|
dm.Tell(DeploymentManagerActor.InstanceSweepTick.Instance);
|
|
await Task.Delay(5);
|
|
}
|
|
});
|
|
|
|
dm.Tell(new DeployInstanceCommand(
|
|
"dep-rev-2", instanceName, "rev-2",
|
|
MakeScriptedConfigJson(instanceName, "var x = 2; var y = x * 3; return y;"),
|
|
"admin", DateTimeOffset.UtcNow));
|
|
var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(20));
|
|
|
|
await ticking.CancelAsync();
|
|
await ticker;
|
|
|
|
Assert.Equal(DeploymentStatus.Success, response.Status);
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName));
|
|
Assert.Empty(SelfHealEvents(siteLog, instanceName));
|
|
}
|
|
|
|
// ── Row 72: a running actor serving a revision the store has replaced ──
|
|
|
|
/// <summary>
|
|
/// THE reproduction of the stale-revision bug. An instance is deployed and running at its first
|
|
/// revision; the durable row is then refreshed to the second revision out of band — what
|
|
/// <c>StoreDeployedConfigIfNewerAsync</c> does when the startup reconcile finds a STALE
|
|
/// config, and what a replicated deploy does on the standby.
|
|
///
|
|
/// <para>
|
|
/// <b>Fails before the fix</b>: the actor keeps serving the first revision's config indefinitely
|
|
/// (the snapshot still shows the old attribute value) and nothing anywhere says the stored
|
|
/// and running revisions differ.
|
|
/// </para>
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Sweep_RestartsInstance_ServingAStaleRevision()
|
|
{
|
|
const string instanceName = "StalePump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog);
|
|
|
|
await DeployAndAwaitAsync(
|
|
dm, instanceName, "rev-1", MakeConfigJson(instanceName, attributeValue: "111"));
|
|
|
|
var beforeHeal = await ResolveInstanceChildAsync(dm, instanceName);
|
|
|
|
// The reconcile refreshes the stale row. Storage only; the running actor is untouched.
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName, attributeValue: "222"),
|
|
"dep-rev-2", "rev-2", true);
|
|
|
|
// The runtime must converge on the stored config...
|
|
await AwaitAssertAsync(async () =>
|
|
{
|
|
var snapshot = await SnapshotAsync(dm, instanceName);
|
|
Assert.False(snapshot.InstanceNotFound);
|
|
Assert.Contains(snapshot.AttributeValues, a =>
|
|
a.AttributeName == "TestAttr" && a.Value?.ToString() == "222");
|
|
}, TimeSpan.FromSeconds(25));
|
|
|
|
// ...through a genuine restart, not an in-place mutation.
|
|
Assert.NotEqual(beforeHeal, await ResolveInstanceChildAsync(dm, instanceName));
|
|
|
|
// ...and the divergence must be surfaced, naming BOTH revisions.
|
|
Assert.Contains(SelfHealEvents(siteLog, instanceName), e =>
|
|
e.Severity == "Warning" &&
|
|
e.Message.Contains("rev-1", StringComparison.Ordinal) &&
|
|
e.Message.Contains("rev-2", StringComparison.Ordinal));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The stale-revision restart must not fire when the stored and running revisions agree —
|
|
/// otherwise every healthy instance on the site would be restarted once a minute.
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Sweep_DoesNotRestart_WhenStoredAndRunningRevisionsMatch()
|
|
{
|
|
const string instanceName = "MatchingRevisionPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog);
|
|
|
|
await DeployAndAwaitAsync(dm, instanceName, "rev-1", MakeConfigJson(instanceName));
|
|
var beforeSweeps = await ResolveInstanceChildAsync(dm, instanceName);
|
|
|
|
// Rewrite the SAME revision — a same-revision re-store must not read as divergence.
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName), "dep-rev-1", "rev-1", true);
|
|
|
|
await Task.Delay(SeveralSweeps);
|
|
|
|
Assert.Equal(beforeSweeps, await ResolveInstanceChildAsync(dm, instanceName));
|
|
Assert.Empty(SelfHealEvents(siteLog, instanceName));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A stale-revision restart that the site compile gate REJECTS must be announced once and
|
|
/// then left alone. Retrying it every tick would be an endless stream of Warning + Error
|
|
/// site events for a config that can never apply, and the running instance is unharmed —
|
|
/// the gate returns before anything is stopped, so it keeps serving the old config.
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task Sweep_StaleRestartRejectedByTheCompileGate_IsAnnouncedOnceAndNotRetried()
|
|
{
|
|
const string instanceName = "UncompilablePump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog);
|
|
|
|
await DeployAndAwaitAsync(dm, instanceName, "rev-1", MakeConfigJson(instanceName));
|
|
var beforeHeal = await ResolveInstanceChildAsync(dm, instanceName);
|
|
|
|
// The refreshed row carries a script that cannot compile.
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName,
|
|
MakeScriptedConfigJson(instanceName, "this is not valid C# at all ((("),
|
|
"dep-rev-bad", "rev-bad", true);
|
|
|
|
// The attempt is announced...
|
|
await AwaitAssertAsync(() =>
|
|
{
|
|
Assert.Single(SelfHealEvents(siteLog, instanceName));
|
|
return Task.CompletedTask;
|
|
}, TimeSpan.FromSeconds(20));
|
|
|
|
// ...its failure is announced...
|
|
await AwaitAssertAsync(() =>
|
|
{
|
|
Assert.Contains(siteLog.OfType("deployment"), e =>
|
|
e.InstanceId == instanceName &&
|
|
e.Severity == "Error" &&
|
|
e.Message.Contains("self-heal failed", StringComparison.OrdinalIgnoreCase));
|
|
return Task.CompletedTask;
|
|
}, TimeSpan.FromSeconds(20));
|
|
|
|
// ...and several further sweeps add nothing and leave the old actor serving.
|
|
await Task.Delay(SeveralSweeps);
|
|
|
|
Assert.Single(SelfHealEvents(siteLog, instanceName));
|
|
Assert.Equal(beforeHeal, await ResolveInstanceChildAsync(dm, instanceName));
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName));
|
|
}
|
|
|
|
// ── Row 101 (config half): a sweep snapshot older than the actor's own state ──
|
|
|
|
/// <summary>
|
|
/// A crafted sweep result standing in for a read that was dispatched earlier than the
|
|
/// generation it is stamped with. <paramref name="generationAtRead"/> <c>0</c> is the value a
|
|
/// read dispatched before this actor had mutated anything would carry; anything the actor has
|
|
/// done since makes it stale. Connection definitions are passed as <see langword="null"/>
|
|
/// (= "the definitions read failed, keep the overlay") so these tests exercise the CONFIG
|
|
/// half of the snapshot in isolation.
|
|
/// </summary>
|
|
private static DeploymentManagerActor.InstanceSweepLoaded SweepSnapshot(
|
|
string instanceName, string revision, string configJson, long generationAtRead,
|
|
bool isEnabled = true) =>
|
|
new(
|
|
[
|
|
new DeployedInstance
|
|
{
|
|
InstanceUniqueName = instanceName,
|
|
ConfigJson = configJson,
|
|
DeploymentId = $"dep-{revision}",
|
|
RevisionHash = revision,
|
|
IsEnabled = isEnabled,
|
|
DeployedAt = DateTimeOffset.UtcNow.ToString("O")
|
|
}
|
|
],
|
|
null,
|
|
ConnectionDefinitions: null,
|
|
StoreGenerationAtRead: generationAtRead);
|
|
|
|
/// <summary>
|
|
/// <b>Deferred register row 101, the stale-revision leg.</b> A sweep read dispatched before a deploy
|
|
/// persisted its new row can land after that deploy has completed AND after the deploy
|
|
/// pipeline released the instance — <see cref="DeploymentManagerActor"/>'s
|
|
/// <c>InstanceIsOwnedByADeployPipeline</c> only covers a deploy still in flight, so it is no
|
|
/// defence here. The snapshot then carries the revision the deploy just REPLACED while the
|
|
/// running actor carries the new one, which is bit-for-bit the divergence the stale-revision repair
|
|
/// looks for — in the wrong direction.
|
|
///
|
|
/// <para>
|
|
/// <b>Fails before the fix</b>: the sweep "repairs" the instance BACKWARD, re-deploying the
|
|
/// snapshot's own superseded config JSON through the redeploy path (attribute back to
|
|
/// <c>111</c>) and announcing it as a self-heal. Nothing retries it either — the per-revision
|
|
/// claim marks the OLD revision as attempted, so the instance simply stays reverted.
|
|
/// </para>
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task StaleSweepSnapshot_CarryingASupersededRevision_DoesNotRestartBackward()
|
|
{
|
|
const string instanceName = "BackwardRestartPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
// Sweep effectively disabled: the crafted message must be the only sweep result in play.
|
|
var dm = CreateBootedDeploymentManager(siteLog, sweepSeconds: 3600);
|
|
|
|
await DeployAndAwaitAsync(
|
|
dm, instanceName, "rev-1", MakeConfigJson(instanceName, attributeValue: "111"));
|
|
await DeployAndAwaitAsync(
|
|
dm, instanceName, "rev-2", MakeConfigJson(instanceName, attributeValue: "222"));
|
|
|
|
var afterDeploys = await ResolveInstanceChildAsync(dm, instanceName);
|
|
|
|
// Generation 0 = captured before either deploy mutated the store.
|
|
dm.Tell(SweepSnapshot(
|
|
instanceName, "rev-1", MakeConfigJson(instanceName, attributeValue: "111"),
|
|
generationAtRead: 0));
|
|
|
|
await Task.Delay(SeveralSweeps);
|
|
|
|
Assert.Empty(SelfHealEvents(siteLog, instanceName));
|
|
Assert.Equal(afterDeploys, await ResolveInstanceChildAsync(dm, instanceName));
|
|
var snapshot = await SnapshotAsync(dm, instanceName);
|
|
Assert.Contains(snapshot.AttributeValues, a =>
|
|
a.AttributeName == "TestAttr" && a.Value?.ToString() == "222");
|
|
}
|
|
|
|
/// <summary>
|
|
/// The stale-snapshot guard must not blunt the stale-revision repair it sits in front of: the SAME message
|
|
/// stamped with the CURRENT generation is a genuine answer and must still converge the
|
|
/// runtime on the stored revision.
|
|
///
|
|
/// <para>
|
|
/// The instance is started by the BOOT LOAD from a pre-written row rather than by a deploy
|
|
/// command, precisely so no store mutation passes through the actor and the current
|
|
/// generation is still <c>0</c> — the same literal the stale test uses, with the opposite
|
|
/// meaning, which is the whole point of the guard.
|
|
/// </para>
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task FreshSweepSnapshot_CarryingANewerRevision_StillRestartsTheInstance()
|
|
{
|
|
const string instanceName = "FreshRestartPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
|
|
await _storage.StoreDeployedConfigAsync(
|
|
instanceName, MakeConfigJson(instanceName, attributeValue: "111"),
|
|
"dep-rev-1", "rev-1", true);
|
|
|
|
var dm = CreateBootedDeploymentManager(siteLog, sweepSeconds: 3600);
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(20));
|
|
|
|
dm.Tell(SweepSnapshot(
|
|
instanceName, "rev-2", MakeConfigJson(instanceName, attributeValue: "222"),
|
|
generationAtRead: 0));
|
|
|
|
await AwaitAssertAsync(async () =>
|
|
{
|
|
var snapshot = await SnapshotAsync(dm, instanceName);
|
|
Assert.False(snapshot.InstanceNotFound);
|
|
Assert.Contains(snapshot.AttributeValues, a =>
|
|
a.AttributeName == "TestAttr" && a.Value?.ToString() == "222");
|
|
}, TimeSpan.FromSeconds(25));
|
|
|
|
Assert.Contains(SelfHealEvents(siteLog, instanceName), e =>
|
|
e.Severity == "Warning" &&
|
|
e.Message.Contains("rev-1", StringComparison.Ordinal) &&
|
|
e.Message.Contains("rev-2", StringComparison.Ordinal));
|
|
}
|
|
|
|
/// <summary>
|
|
/// <b>Deferred register row 101, the missing-start leg.</b> A sweep read dispatched before a delete
|
|
/// committed still shows the instance's row — enabled, with a config — while the actor has
|
|
/// already stopped it and dropped its bookkeeping. The delete specifically clears
|
|
/// <c>_sweepAttemptedRevisions</c>, so the once-per-revision claim is freshly available and
|
|
/// does not stand in the way either.
|
|
///
|
|
/// <para>
|
|
/// <b>Fails before the fix</b>: the sweep RESURRECTS the instance the operator just deleted,
|
|
/// re-adds it to the deployed-name set, and announces the resurrection as a self-heal.
|
|
/// </para>
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task StaleSweepSnapshot_ShowingADeletedInstance_DoesNotResurrectIt()
|
|
{
|
|
const string instanceName = "ResurrectedPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog, sweepSeconds: 3600);
|
|
|
|
await DeployAndAwaitAsync(dm, instanceName, "rev-1", MakeConfigJson(instanceName));
|
|
|
|
dm.Tell(new DeleteInstanceCommand("cmd-del-1", instanceName, DateTimeOffset.UtcNow));
|
|
Assert.True(ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(15)).Success);
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.False(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(15));
|
|
|
|
// Generation 0 = captured before the deploy and the delete mutated the store.
|
|
dm.Tell(SweepSnapshot(
|
|
instanceName, "rev-1", MakeConfigJson(instanceName), generationAtRead: 0));
|
|
|
|
await Task.Delay(SeveralSweeps);
|
|
|
|
Assert.False(await InstanceIsRunningAsync(dm, instanceName));
|
|
Assert.Empty(SelfHealEvents(siteLog, instanceName));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The mirror of the resurrection test: the same enabled-config-with-no-actor snapshot,
|
|
/// stamped with the CURRENT generation, must still perform the missing-start heal. Nothing has
|
|
/// mutated the store through this actor, so <c>0</c> is current here.
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task FreshSweepSnapshot_ShowingAStoredEnabledInstance_StillStartsIt()
|
|
{
|
|
const string instanceName = "FreshStartPump";
|
|
var siteLog = new FakeSiteEventLogger();
|
|
var dm = CreateBootedDeploymentManager(siteLog, sweepSeconds: 3600);
|
|
|
|
dm.Tell(SweepSnapshot(
|
|
instanceName, "rev-1", MakeConfigJson(instanceName), generationAtRead: 0));
|
|
|
|
await AwaitAssertAsync(async () =>
|
|
Assert.True(await InstanceIsRunningAsync(dm, instanceName)), TimeSpan.FromSeconds(20));
|
|
Assert.Single(SelfHealEvents(siteLog, instanceName));
|
|
}
|
|
|
|
/// <summary>
|
|
/// <b>Row 101, boot half.</b> The boot-time load has to COMPLETE — a node whose startup read
|
|
/// is simply dropped stays a silent zero-instance node — so a superseded boot snapshot is
|
|
/// re-read rather than discarded like the sweep's.
|
|
///
|
|
/// <para>
|
|
/// The race is made deterministic by a slow first read: a lifecycle command that mutates
|
|
/// <c>deployed_configurations</c> is processed while it is in flight, so the snapshot lands
|
|
/// stamped with a generation the actor has already moved past. The observable is the read
|
|
/// itself — the loader must be called a SECOND time — and, just as importantly, exactly twice:
|
|
/// the re-dispatch captures the post-mutation generation, so it converges instead of looping.
|
|
/// </para>
|
|
///
|
|
/// <para><b>Fails before the fix</b>: one read, and the pre-mutation snapshot is applied.</para>
|
|
/// </summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task StaleBootSnapshot_IsReReadRatherThanApplied()
|
|
{
|
|
var reads = 0;
|
|
var health = new DeploymentManagerStartupProbe();
|
|
|
|
var configLoader = SlowFirstConfigLoad(() => Interlocked.Increment(ref reads));
|
|
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
|
|
_storage,
|
|
_compilationService,
|
|
_sharedScriptLibrary,
|
|
null,
|
|
new SiteRuntimeOptions { InstanceSweepIntervalSeconds = 3600 },
|
|
NullLogger<DeploymentManagerActor>.Instance,
|
|
dclManager: null,
|
|
healthCollector: health,
|
|
serviceProvider: null,
|
|
loggerFactory: null,
|
|
configFetcher: null,
|
|
startupLoadRetryInterval: null,
|
|
configLoader: configLoader,
|
|
externalSystemCache: null,
|
|
scriptScheduler: null,
|
|
connectionDefinitionLoader: null)));
|
|
|
|
// Mutates deployed_configurations (a no-op UPDATE for an unknown instance is still a
|
|
// write dispatched from the actor thread) while the first read is still in flight.
|
|
dm.Tell(new DisableInstanceCommand("cmd-boot-race", "NoSuchInstance", DateTimeOffset.UtcNow));
|
|
ExpectMsg<InstanceLifecycleResponse>(TimeSpan.FromSeconds(15));
|
|
|
|
await AwaitAssertAsync(() =>
|
|
{
|
|
Assert.Equal(2, Volatile.Read(ref reads));
|
|
return Task.CompletedTask;
|
|
}, TimeSpan.FromSeconds(20));
|
|
|
|
// The second read is fresh, so the boot completes and stops re-reading...
|
|
await Task.Delay(TimeSpan.FromSeconds(2));
|
|
Assert.Equal(2, Volatile.Read(ref reads));
|
|
|
|
// ...and a LATE superseded snapshot (the S5 retry timer racing a success) is dropped
|
|
// rather than triggering another read, because the boot it would complete is done.
|
|
dm.Tell(new DeploymentManagerActor.StartupConfigsLoaded(
|
|
[], null, ConnectionDefinitions: null, StoreGenerationAtRead: -1));
|
|
await Task.Delay(TimeSpan.FromSeconds(2));
|
|
Assert.Equal(2, Volatile.Read(ref reads));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A deployed-config loader whose FIRST call takes long enough for a lifecycle command to be
|
|
/// processed while it is in flight; later calls answer immediately. Always returns an empty
|
|
/// site — these tests assert on how many reads happen, not on what they carry.
|
|
/// </summary>
|
|
private static Func<Task<List<DeployedInstance>>> SlowFirstConfigLoad(Action onRead)
|
|
{
|
|
var calls = 0;
|
|
return async () =>
|
|
{
|
|
onRead();
|
|
if (Interlocked.Increment(ref calls) == 1)
|
|
await Task.Delay(TimeSpan.FromSeconds(2));
|
|
return [];
|
|
};
|
|
}
|
|
}
|