using Akka.Actor; using Akka.TestKit.Xunit2; using Microsoft.AspNetCore.DataProtection; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Types; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates; using ZB.MOM.WW.ScadaBridge.Communication; using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase; using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories; using ZB.MOM.WW.ScadaBridge.DeploymentManager; using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening; namespace ZB.MOM.WW.ScadaBridge.IntegrationTests; /// /// P7-5: a bulk site deploy runs all three phases over ONE scoped /// DbContext, so a persistent SaveChanges failure on a single /// instance used to fail every instance that came after it. /// /// /// Why this needs a real EF context. The mechanism is not in the service's /// control flow, it is in EF's: a failed SaveChanges does NOT untrack the /// entity that caused it. The entity stays in the change set and is re-attempted /// by the very next save made through the same context — including the cleanup /// save inside MarkDeploymentFailedAsync, whose whole job is to stop a /// record being stranded InProgress. A mocked repository could not prove /// that; a real over SQLite does. /// /// /// /// Honesty note on the injected failure. The persistent failure itself is /// injected by a save interceptor rather than provoked by a real constraint /// violation, because the entity that must fail (the row /// the deploy path mutates) is created and mutated inside the service, where a /// test cannot reach in and corrupt it mid-batch. The interceptor throws /// at exactly the point a constraint violation, /// a concurrency conflict or a dropped connection would — everything after that, /// including the cascade and the fix, is real EF behaviour. /// /// public sealed class BulkDeployPoisonedContextTests : TestKit, IDisposable { private const int InstanceCount = 4; private readonly SqliteConnection _connection; private readonly ScadaBridgeDbContext _context; private readonly PoisonedEntityInterceptor _poison = new(); private int _siteId; public BulkDeployPoisonedContextTests() { _connection = new SqliteConnection("DataSource=:memory:"); _connection.Open(); var options = new DbContextOptionsBuilder() .UseSqlite(_connection) .ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning)) .AddInterceptors(_poison) .Options; _context = new SqliteContext(options); _context.Database.EnsureCreated(); Seed(); } /// /// The whole finding in one run: instance 1's post-success persistence fails /// permanently (best-effort, so the deploy still reports Success), which leaves /// its row poisoning the shared change tracker. Every /// later instance's status write then rides on that same change set. /// /// /// Pre-fix this cascaded: instances 2, 3 and 4 all failed, and their records /// stayed InProgress because the cleanup save re-attempted — and /// re-failed on — the same poisoned entity. Post-fix the cleanup detaches the /// pending work, so instance 2 is the only casualty (correctly recorded /// Failed, not stranded), and 3 and 4 save cleanly. /// /// [Fact] public async Task BulkDeploy_PersistentSaveFailureOnOneInstance_DoesNotCascadeToTheRest() { _poison.PoisonInstanceNamed("Inst-01"); var service = CreateService(); var result = await service.DeploySiteAsync(_siteId, "admin"); Assert.True(result.IsSuccess); var summary = result.Value; Assert.Equal(InstanceCount, summary.InstanceResults.Count); // Instances 3 and 4 are downstream of the poison and must be unaffected: // this is the assertion that fails outright without the detach. foreach (var name in new[] { "Inst-03", "Inst-04" }) { var row = Assert.Single(summary.InstanceResults, r => r.UniqueName == name); Assert.True(row.Success, $"{name} failed with: {row.ErrorMessage}"); } // Instance 2 is the one collateral casualty — its own status write shared // the poisoned change set — but it must be recorded Failed, never left // InProgress, which is what the retry on the cleaned tracker buys. var stranded = await NoLongerInProgressAsync(); Assert.Empty(stranded); // ...and the fix is not "swallow everything": the poisoned instance's own // side effects genuinely did not persist, so its Instance row is still // NotDeployed even though the site confirmed the apply. That divergence is // the pre-existing best-effort contract, logged for operator reconciliation. await using var verify = NewContext(); Assert.Equal( InstanceState.NotDeployed, verify.Instances.Single(i => i.UniqueName == "Inst-01").State); } /// /// The narrow EF fact the fix rests on, pinned on its own so a future reader /// does not have to re-derive it: a failed save leaves its entity tracked and /// the NEXT save re-attempts it, and /// IDeploymentManagerRepository.DetachPendingChanges is what clears it. /// [Fact] public async Task FailedSave_KeepsTheEntityTracked_UntilPendingChangesAreDetached() { var repository = new DeploymentManagerRepository(_context); var instance = _context.Instances.Single(i => i.UniqueName == "Inst-01"); _poison.PoisonInstanceNamed("Inst-01"); instance.State = InstanceState.Enabled; await repository.UpdateInstanceAsync(instance); await Assert.ThrowsAsync(() => repository.SaveChangesAsync()); // An unrelated, perfectly valid write now fails too — the poisoned entity // is still in the change set. var record = new DeploymentRecord("dep-unrelated", "admin") { InstanceId = _context.Instances.Single(i => i.UniqueName == "Inst-02").Id, Status = DeploymentStatus.Failed, RevisionHash = "sha256:x", DeployedAt = DateTimeOffset.UtcNow, }; await repository.AddDeploymentRecordAsync(record); await Assert.ThrowsAsync(() => repository.SaveChangesAsync()); // Detaching the pending work un-poisons the unit of work; the same write // then succeeds. Assert.True(repository.DetachPendingChanges() > 0); await repository.AddDeploymentRecordAsync(record); Assert.Equal(1, await repository.SaveChangesAsync()); } /// Deployment records left non-terminal after the batch — the stranded-InProgress check. private async Task> NoLongerInProgressAsync() { await using var verify = NewContext(); return await verify.DeploymentRecords .Where(r => r.Status == DeploymentStatus.InProgress || r.Status == DeploymentStatus.Pending) .Select(r => r.DeploymentId) .ToListAsync(); } private DeploymentService CreateService() { var repository = new DeploymentManagerRepository(_context); var siteRepository = new SiteRepository(_context); var pipeline = Substitute.For(); pipeline.CreateSession().Returns(_ => new FlattenSession(new TemplateGraphWatermark())); pipeline .FlattenAndValidateAsync( Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(ci => Result.Success( new FlatteningPipelineResult( new FlattenedConfiguration { InstanceUniqueName = $"Inst-{ci.ArgAt(0):00}" }, $"sha256:{ci.ArgAt(0)}", ValidationResult.Success()))); var comms = new CommunicationService( Options.Create(new CommunicationOptions { DeploymentTimeout = TimeSpan.FromSeconds(10) }), NullLogger.Instance); comms.SetCommunicationActor(Sys.ActorOf(Props.Create(() => new AcceptingSiteActor()))); // Null probe result = "staleness indeterminate", so the up-to-date // pre-filter never skips: this test is about the persistence cascade and // needs every instance to actually run. var probe = Substitute.For(); probe.GetCurrentRevisionHashAsync(Arg.Any(), Arg.Any()).Returns((string?)null); return new DeploymentService( repository, siteRepository, pipeline, comms, new OperationLockManager(), Substitute.For(), new DiffService(), new RevisionHashService(), new DeploymentStatusNotifier(NullLogger.Instance), probe, Options.Create(new DeploymentManagerOptions { OperationLockTimeout = TimeSpan.FromSeconds(5), SiteDeploymentMaxParallelism = 1, SiteDeploymentTimeoutPerInstance = TimeSpan.FromSeconds(10), SiteDeploymentReconcileTimeout = TimeSpan.FromSeconds(2), }), Options.Create(new CommunicationOptions { CentralFetchBaseUrl = "https://central.test:9000", PendingDeploymentTtl = TimeSpan.FromMinutes(5), }), NullLogger.Instance); } private void Seed() { using var ctx = NewContext(); var site = new Site("Poison Site", "site-poison"); var template = new Template("T-Poison"); ctx.Sites.Add(site); ctx.Templates.Add(template); ctx.SaveChanges(); _siteId = site.Id; for (var i = 1; i <= InstanceCount; i++) { ctx.Instances.Add(new Instance($"Inst-{i:00}") { SiteId = site.Id, TemplateId = template.Id, State = InstanceState.NotDeployed, }); } ctx.SaveChanges(); } private ScadaBridgeDbContext NewContext() => new SqliteContext(new DbContextOptionsBuilder() .UseSqlite(_connection) .ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning)) .Options); public new void Dispose() { _context.Dispose(); _connection.Dispose(); base.Dispose(); } /// /// Fails every SaveChanges whose change set contains the named /// — a stand-in for a permanent constraint violation on /// one tracked entity, raised at the point EF raises a real one. /// private sealed class PoisonedEntityInterceptor : SaveChangesInterceptor { private string? _poisonedUniqueName; public void PoisonInstanceNamed(string uniqueName) => _poisonedUniqueName = uniqueName; public override InterceptionResult SavingChanges( DbContextEventData eventData, InterceptionResult result) { ThrowIfPoisoned(eventData); return base.SavingChanges(eventData, result); } public override ValueTask> SavingChangesAsync( DbContextEventData eventData, InterceptionResult result, CancellationToken cancellationToken = default) { ThrowIfPoisoned(eventData); return base.SavingChangesAsync(eventData, result, cancellationToken); } private void ThrowIfPoisoned(DbContextEventData eventData) { if (_poisonedUniqueName is null || eventData.Context is null) return; var poisoned = eventData.Context.ChangeTracker.Entries() .Any(e => e.State != EntityState.Unchanged && e.State != EntityState.Detached && e.Entity.UniqueName == _poisonedUniqueName); if (poisoned) throw new DbUpdateException($"simulated persistent failure on {_poisonedUniqueName}"); } } /// Answers every deployment with Success, so only persistence can fail here. private sealed class AcceptingSiteActor : ReceiveActor { public AcceptingSiteActor() { Receive(env => { if (env.Message is not RefreshDeploymentCommand cmd) return; Sender.Tell(new DeploymentStatusResponse( cmd.DeploymentId, cmd.InstanceUniqueName, DeploymentStatus.Success, null, DateTimeOffset.UtcNow)); }); } } /// /// SQLite-adapted context: SQLite has no rowversion and no native /// DateTimeOffset. Mirrors the adaptation DeployNotifyFetchTests uses. /// private sealed class SqliteContext : ScadaBridgeDbContext { public SqliteContext(DbContextOptions options) : base(options, new EphemeralDataProtectionProvider()) { } protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); modelBuilder.Entity(builder => { builder.Property(d => d.RowVersion) .IsRequired(false) .IsConcurrencyToken(false) .ValueGeneratedNever(); }); var converter = new ValueConverter( v => v.UtcDateTime.ToString("o"), v => DateTimeOffset.Parse(v)); var nullableConverter = new ValueConverter( v => v.HasValue ? v.Value.UtcDateTime.ToString("o") : null, v => v != null ? DateTimeOffset.Parse(v) : null); foreach (var entityType in modelBuilder.Model.GetEntityTypes()) { foreach (var property in entityType.GetProperties()) { if (property.ClrType == typeof(DateTimeOffset)) { property.SetValueConverter(converter); property.SetColumnType("TEXT"); } else if (property.ClrType == typeof(DateTimeOffset?)) { property.SetValueConverter(nullableConverter); property.SetColumnType("TEXT"); } } } } } }