feat(controlplane): deployment artifact retention on the admin singleton
Every deploy stored the whole gzip'd fleet snapshot in Deployment.ArtifactBlob forever (R03:1.2). DeploymentRetentionActor (admin singleton, proxy-less) sweeps every Deployments:Retention: IntervalMinutes (60): terminal rows (Sealed / PartiallyFailed / TimedOut) older than the newest KeepSealed (50) and not referenced by a NodeDeploymentState started/applied within ReferencedDays (30) get their blob EMPTIED — never deleted, never NULLed: the column stays NOT NULL and every reader already treats zero bytes as "no artifact" (#485), so no schema migration and no reader change. In-flight rows are never touched; MaxRowsPerSweep (200) bounds one sweep. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,174 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// R03:1.2 — every deploy stored the whole fleet snapshot forever. The retention sweep empties the
|
||||
/// blob on old terminal rows (never deletes a row, never touches an in-flight one, never the newest N,
|
||||
/// never one a node applied recently).
|
||||
/// </summary>
|
||||
public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase
|
||||
{
|
||||
private static readonly DateTime SweepNow = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact()
|
||||
{
|
||||
var f = NewInMemoryDbFactory();
|
||||
var ids = Seed(f, sealedCount: 12); // ids[0] newest … ids[11] oldest
|
||||
var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 };
|
||||
|
||||
int pruned;
|
||||
await using (var db = f.CreateDbContext())
|
||||
pruned = await DeploymentRetentionActor.SweepAsync(db, options, SweepNow);
|
||||
|
||||
pruned.ShouldBe(7);
|
||||
await using var verify = f.CreateDbContext();
|
||||
foreach (var id in ids.Take(5))
|
||||
(await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.Length.ShouldBeGreaterThan(0, "newest 5 keep their blob");
|
||||
foreach (var id in ids.Skip(5))
|
||||
(await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.ShouldBeEmpty("older rows are emptied");
|
||||
// Rows survive: the audit record is intact.
|
||||
(await verify.Deployments.CountAsync()).ShouldBe(12);
|
||||
|
||||
// Idempotent: a second sweep finds nothing.
|
||||
await using var again = f.CreateDbContext();
|
||||
(await DeploymentRetentionActor.SweepAsync(again, options, SweepNow)).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_keeps_a_row_a_node_applied_recently_and_never_touches_an_in_flight_row()
|
||||
{
|
||||
var f = NewInMemoryDbFactory();
|
||||
var ids = Seed(f, sealedCount: 4);
|
||||
var inFlight = Guid.NewGuid();
|
||||
await using (var db = f.CreateDbContext())
|
||||
{
|
||||
db.ClusterNodes.Add(new ClusterNode
|
||||
{
|
||||
NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1",
|
||||
ApplicationUri = "urn:x", Enabled = true, CreatedBy = "test",
|
||||
});
|
||||
// ids[3] is the oldest; a node applied it yesterday, so it stays.
|
||||
db.NodeDeploymentStates.Add(new NodeDeploymentState
|
||||
{
|
||||
NodeId = "site-a-1:4053", DeploymentId = ids[3], Status = NodeDeploymentStatus.Applied,
|
||||
StartedAtUtc = SweepNow.AddDays(-1), AppliedAtUtc = SweepNow.AddDays(-1),
|
||||
});
|
||||
// ids[2]: applied long ago — not protected.
|
||||
db.NodeDeploymentStates.Add(new NodeDeploymentState
|
||||
{
|
||||
NodeId = "site-a-1:4053", DeploymentId = ids[2], Status = NodeDeploymentStatus.Applied,
|
||||
StartedAtUtc = SweepNow.AddDays(-90), AppliedAtUtc = SweepNow.AddDays(-90),
|
||||
});
|
||||
// An ancient in-flight row with a blob — never touched.
|
||||
db.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = inFlight, RevisionHash = new string('f', 64), Status = DeploymentStatus.AwaitingApplyAcks,
|
||||
CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-400), ArtifactBlob = new byte[] { 1, 2, 3 },
|
||||
});
|
||||
await db.SaveChangesAsync();
|
||||
}
|
||||
var options = new DeploymentRetentionOptions { KeepSealed = 1, ReferencedDays = 30 };
|
||||
|
||||
await using (var db = f.CreateDbContext())
|
||||
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); // ids[1], ids[2]
|
||||
|
||||
await using var verify = f.CreateDbContext();
|
||||
(await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty("newest");
|
||||
(await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[1])).ArtifactBlob.ShouldBeEmpty();
|
||||
(await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[2])).ArtifactBlob.ShouldBeEmpty("applied 90 days ago");
|
||||
(await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[3])).ArtifactBlob.ShouldNotBeEmpty("applied yesterday");
|
||||
(await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls()
|
||||
{
|
||||
var f = NewInMemoryDbFactory();
|
||||
Seed(f, sealedCount: 10);
|
||||
var options = new DeploymentRetentionOptions { KeepSealed = 2, MaxRowsPerSweep = 3 };
|
||||
|
||||
await using (var db = f.CreateDbContext())
|
||||
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3);
|
||||
await using (var db = f.CreateDbContext())
|
||||
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3);
|
||||
await using (var db = f.CreateDbContext())
|
||||
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2);
|
||||
await using (var db = f.CreateDbContext())
|
||||
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>The actor runs the same rule off its thread and reports back.</summary>
|
||||
[Fact]
|
||||
public void Actor_sweeps_on_request_and_reports_the_count()
|
||||
{
|
||||
var f = NewInMemoryDbFactory();
|
||||
Seed(f, sealedCount: 6);
|
||||
var actor = Sys.ActorOf(DeploymentRetentionActor.Props(f, new DeploymentRetentionOptions
|
||||
{
|
||||
KeepSealed = 4, InitialDelaySeconds = 3600, IntervalMinutes = 60,
|
||||
}));
|
||||
|
||||
actor.Tell(DeploymentRetentionActor.Sweep.Instance);
|
||||
|
||||
ExpectMsg<DeploymentRetentionActor.SweepCompleted>(PresenceBudget).Pruned.ShouldBe(2);
|
||||
using var verify = f.CreateDbContext();
|
||||
verify.Deployments.Count(d => d.ArtifactBlob.Length == 0).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Options_bind_from_configuration_with_defaults()
|
||||
{
|
||||
var defaults = DeploymentRetentionOptions.Bind(null);
|
||||
defaults.Enabled.ShouldBeTrue();
|
||||
defaults.KeepSealed.ShouldBe(50);
|
||||
defaults.ReferencedDays.ShouldBe(30);
|
||||
defaults.IntervalMinutes.ShouldBe(60);
|
||||
|
||||
var config = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Deployments:Retention:KeepSealed"] = "7",
|
||||
["Deployments:Retention:Enabled"] = "false",
|
||||
})
|
||||
.Build();
|
||||
var bound = DeploymentRetentionOptions.Bind(config);
|
||||
bound.KeepSealed.ShouldBe(7);
|
||||
bound.Enabled.ShouldBeFalse();
|
||||
bound.ReferencedDays.ShouldBe(30);
|
||||
}
|
||||
|
||||
/// <summary>Seeds sealed deployments, newest first; each one hour older than the previous.</summary>
|
||||
private static List<Guid> Seed(IDbContextFactory<OtOpcUaConfigDbContext> f, int sealedCount)
|
||||
{
|
||||
var ids = new List<Guid>();
|
||||
using var db = f.CreateDbContext();
|
||||
for (var i = 0; i < sealedCount; i++)
|
||||
{
|
||||
var id = Guid.NewGuid();
|
||||
ids.Add(id);
|
||||
db.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = id,
|
||||
RevisionHash = new string('a', 64),
|
||||
Status = i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed,
|
||||
CreatedBy = "test",
|
||||
CreatedAtUtc = SweepNow.AddHours(-i),
|
||||
SealedAtUtc = SweepNow.AddHours(-i),
|
||||
ArtifactBlob = new byte[] { 0x1f, 0x8b, (byte)i },
|
||||
});
|
||||
}
|
||||
db.SaveChanges();
|
||||
return ids;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user