fix(deployment): audit-write fault can no longer flip a committed Success deployment to Failed

Route all post-terminal-status lifecycle audit writes (Deploy/Disable/Enable/Delete/DeleteOrphaned) through a swallow-and-warn TryLogAuditAsync helper so a failed best-effort audit write can neither reach the outer catch (flipping Success to Failed) nor propagate to the caller.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
This commit is contained in:
Joseph Doherty
2026-07-09 16:12:56 -04:00
parent 5c52b4c80b
commit 721a2e595f
2 changed files with 70 additions and 6 deletions
@@ -326,8 +326,11 @@ public class DeploymentService
forceEnabledState: true, cancellationToken);
}
// Audit log
await _auditService.LogAsync(user, "Deploy", "Instance", instanceId.ToString(),
// Audit log. Best-effort and post-terminal: the deployment record's
// status is already committed above, so a failed audit write must NOT
// fall through to the outer catch and flip a committed Success back to
// Failed (audit is best-effort — the action's own result is authoritative).
await TryLogAuditAsync(user, "Deploy", "Instance", instanceId.ToString(),
instance.UniqueName, new { DeploymentId = deploymentId, Status = record.Status.ToString() },
cancellationToken);
@@ -458,7 +461,7 @@ public class DeploymentService
await _repository.SaveChangesAsync(cancellationToken);
}
await _auditService.LogAsync(user, "Disable", "Instance", instanceId.ToString(),
await TryLogAuditAsync(user, "Disable", "Instance", instanceId.ToString(),
instance.UniqueName, new { CommandId = commandId, response.Success },
cancellationToken);
@@ -522,7 +525,7 @@ public class DeploymentService
await _repository.SaveChangesAsync(cancellationToken);
}
await _auditService.LogAsync(user, "Enable", "Instance", instanceId.ToString(),
await TryLogAuditAsync(user, "Enable", "Instance", instanceId.ToString(),
instance.UniqueName, new { CommandId = commandId, response.Success },
cancellationToken);
@@ -609,7 +612,7 @@ public class DeploymentService
"removed -- the central record is now orphaned and must be reconciled manually",
instance.UniqueName);
await _auditService.LogAsync(user, "DeleteOrphaned", "Instance", instanceId.ToString(),
await TryLogAuditAsync(user, "DeleteOrphaned", "Instance", instanceId.ToString(),
instance.UniqueName, new { CommandId = commandId, Error = ex.Message },
CancellationToken.None);
@@ -619,7 +622,7 @@ public class DeploymentService
}
}
await _auditService.LogAsync(user, "Delete", "Instance", instanceId.ToString(),
await TryLogAuditAsync(user, "Delete", "Instance", instanceId.ToString(),
instance.UniqueName, new { CommandId = commandId, response.Success },
cancellationToken);
@@ -1035,6 +1038,37 @@ public class DeploymentService
/// <param name="instanceUniqueName">The instance unique name used as the audit target name.</param>
/// <param name="commandId">The lifecycle command's correlation id, so the audit entry can be matched to logs.</param>
/// <param name="timeoutException">The captured <see cref="TimeoutException"/> or <see cref="OperationCanceledException"/>.</param>
/// <summary>
/// Best-effort audit write for a post-terminal-status action. By the time this
/// runs, the deployment/lifecycle action has already committed its authoritative
/// result; a failed audit write must be logged for operator reconciliation but
/// must NEVER surface to the caller or flip the committed status — "audit-write
/// failure NEVER aborts the user-facing action" (Centralized Audit Log design).
/// Any exception is swallowed and logged at Warning.
/// </summary>
private async Task TryLogAuditAsync(
string user,
string action,
string entityType,
string entityId,
string entityName,
object? afterState,
CancellationToken cancellationToken)
{
try
{
await _auditService.LogAsync(
user, action, entityType, entityId, entityName, afterState, cancellationToken);
}
catch (Exception auditEx)
{
_logger.LogWarning(auditEx,
"Failed to write {Action} audit entry for {EntityType} {EntityName} -- the action " +
"itself succeeded and its committed status is authoritative; audit is best-effort",
action, entityType, entityName);
}
}
private async Task TryLogLifecycleTimeoutAsync(
string user,
string action,
@@ -878,6 +878,36 @@ public class DeploymentServiceTests : TestKit
Arg.Any<object>(), Arg.Any<CancellationToken>());
}
[Fact]
public async Task DeployInstanceAsync_DeployAuditWriteThrows_KeepsCommittedSuccess()
{
// #05-T17: the post-success "Deploy" audit write is best-effort. If it
// throws, the already-committed Success record must NOT be flipped to
// Failed by the outer catch — "audit-write failure NEVER aborts the
// user-facing action".
var instance = new Instance("AuditFaultInst") { Id = 52, SiteId = 1, State = InstanceState.NotDeployed };
_repo.GetInstanceByIdAsync(52, Arg.Any<CancellationToken>()).Returns(instance);
SetupValidPipeline(52, "AuditFaultInst", "sha256:target");
_repo.GetCurrentDeploymentStatusAsync(52, Arg.Any<CancellationToken>())
.Returns((DeploymentRecord?)null);
// The Deploy audit LogAsync faults.
_audit.When(a => a.LogAsync(
Arg.Any<string>(), "Deploy", Arg.Any<string>(), Arg.Any<string>(),
Arg.Any<string>(), Arg.Any<object>(), Arg.Any<CancellationToken>()))
.Do(_ => throw new InvalidOperationException("audit pipeline down"));
var counters = new ReconcileProbeCounters();
var commActor = Sys.ActorOf(Props.Create(() =>
new ReconcileProbeActor(counters, siteHash: "sha256:target", failQuery: false)));
var service = CreateServiceWithCommActor(commActor);
var result = await service.DeployInstanceAsync(52, "admin");
Assert.True(result.IsSuccess);
Assert.Equal(DeploymentStatus.Success, result.Value.Status);
}
// ── Telemetry follow-on: scadabridge.deployments.applied on deploy success ──
[Fact]