dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(site): artifact apply writes deployed secrets and answers SecretsApplied

Browse Source
This commit is contained in:
Joseph Doherty
2026-09-28 19:29:55 -04:00
parent 838fb4177b
commit 52cd4fb096
5 changed files with 466 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs
+8 -1
View File
@@ -836,13 +836,20 @@ akka {{
// it after an artifact deploy applies external-system changes.
var externalSystemCache = _serviceProvider
.GetService<ZB.MOM.WW.ScadaBridge.SiteRuntime.Repositories.ExternalSystemDefinitionCache>();
// Task 10 (site-secrets artifact deploy): the same per-node singleton AddSiteRuntime
// registers (and AddHostedService runs periodically) is reconciled SYNCHRONOUSLY by
// HandleDeployArtifacts right after it writes a shipped secret set, so the deploy
// reply's SecretsApplied reflects this node's own apply/verify outcome immediately
// rather than waiting for the next periodic pass.
var secretsMirror = _serviceProvider
.GetService<ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets.DeployedSecretsMirror>();
var dm = SingletonRegistrar.Start(
_actorSystem!, "deployment-manager",
Props.Create(() => new DeploymentManagerActor(
storage, compilationService, sharedScriptLibrary, streamManager,
siteRuntimeOptionsValue, dmLogger, dclManager,
siteHealthCollector, _serviceProvider, null, deploymentConfigFetcher,
null, null, externalSystemCache)),
null, null, externalSystemCache, null, null, null, secretsMirror)),
_logger, role: siteRole);
var dmProxy = dm.Proxy;
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs
+92 -2
View File
@@ -18,6 +18,7 @@ using ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Repositories;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
@@ -49,6 +50,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// external-system caching — the invalidation call is then simply skipped.
/// </summary>
private readonly ExternalSystemDefinitionCache? _externalSystemCache;
/// <summary>
/// Task 10 (site-secrets artifact deploy): per-node mirror reconciled synchronously by
/// <see cref="HandleDeployArtifacts"/> right after the shipped secrets are written to
/// <c>deployed_secrets</c>, so the reply's <c>SecretsApplied</c> reflects this node's own
/// apply/verify outcome rather than the next periodic pass. Optional/null on nodes/tests
/// that never register secret deployment — a shipped <c>command.Secrets</c> with no mirror
/// registered fails the deploy (see <see cref="HandleDeployArtifacts"/>).
/// </summary>
private readonly DeployedSecretsMirror? _secretsMirror;
private readonly SiteStreamManager? _streamManager;
private readonly SiteRuntimeOptions _options;
private readonly ILogger<DeploymentManagerActor> _logger;
@@ -306,6 +316,12 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
/// ensure-started heal uses; defaults to reading from <paramref name="storage"/>. Primarily
/// for tests that need to hold the read open mid-flight to drive the row-101 race.
/// </param>
/// <param name="secretsMirror">
/// Task 10 (site-secrets artifact deploy): this node's <see cref="DeployedSecretsMirror"/>,
/// reconciled synchronously by <see cref="HandleDeployArtifacts"/> right after a shipped
/// <c>command.Secrets</c> set is written to <c>deployed_secrets</c>. Optional/null on
/// nodes/tests that never register secret deployment.
/// </param>
public DeploymentManagerActor(
SiteStorageService storage,
ScriptCompilationService compilationService,
@@ -323,9 +339,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
ExternalSystemDefinitionCache? externalSystemCache = null,
ScriptExecutionScheduler? scriptScheduler = null,
Func<Task<List<StoredDataConnectionDefinition>>>? connectionDefinitionLoader = null,
Func<string, Task<DeployedInstance?>>? deployedConfigLoader = null)
Func<string, Task<DeployedInstance?>>? deployedConfigLoader = null,
DeployedSecretsMirror? secretsMirror = null)
{
_scriptScheduler = scriptScheduler;
_secretsMirror = secretsMirror;
_storage = storage;
_compilationService = compilationService;
_deployCompileValidator = new DeployCompileValidator(compilationService);
@@ -3317,10 +3335,82 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// SMTP configuration is
// central-only and is never stored on a site (see the purge above).
// Site-secrets artifact deploy (Task 10): write the shipped secrets to the
// replicated deployed_secrets table, then reconcile THIS node's own mirror
// synchronously — the reply's SecretsApplied must reflect this node's own
// apply/verify outcome, not whatever the next periodic pass finds. A null
// command.Secrets means "central didn't ship secrets" (old-central
// compatible) and touches nothing; a non-null Secrets with no mirror
// registered on this node fails the deploy rather than silently applying
// nothing.
int? secretsApplied = null;
string? secretError = null;
if (command.Secrets != null)
{
if (_secretsMirror == null)
{
if (command.Secrets.Count > 0)
{
secretError = "Secret deployment is not enabled on this site node";
}
else
{
secretsApplied = 0;
}
}
else
{
_logger.LogInformation(
"Applying {Count} deployed secret(s), deploymentId={DeploymentId}",
command.Secrets.Count, command.DeploymentId);
await _storage.UpsertDeployedSecretsAsync(command.Secrets, command.DeploymentId);
var mirror = await _secretsMirror.ReconcileAsync(CancellationToken.None);
// The mirror reconciles the WHOLE site-wide deployed_secrets table
// every pass — not only the names THIS deploy just shipped — so a
// failure unrelated to this deploy's own secrets (another instance's
// already-deployed secret still failing, or the removal of a
// no-longer-referenced one) must never fail THIS response or show up
// in its ErrorMessage. Only failures among the SHIPPED names count.
var shippedNames = new HashSet<string>(
command.Secrets.Select(s => s.Name), StringComparer.Ordinal);
var shippedFailureMessages = new List<string>();
var shippedFailedNames = new HashSet<string>(StringComparer.Ordinal);
for (var i = 0; i < mirror.FailedNames.Count; i++)
{
var failedName = mirror.FailedNames[i];
var failedMessage = mirror.Failed[i];
if (shippedNames.Contains(failedName))
{
shippedFailedNames.Add(failedName);
shippedFailureMessages.Add(failedMessage);
}
else
{
_logger.LogWarning(
"Deployed secrets mirror reported a failure unrelated to " +
"this deploy ({DeploymentId}): {Failure}",
command.DeploymentId, failedMessage);
}
}
// Never negative (ruling): shippedFailedNames is a set intersected
// against the shipped names, so this can only go below zero if the
// shipped list itself contains duplicate names.
secretsApplied = Math.Max(0, command.Secrets.Count - shippedFailedNames.Count);
if (shippedFailureMessages.Count > 0)
{
secretError = string.Join("; ", shippedFailureMessages);
}
}
}
// Replicate artifacts to standby node
return new ArtifactDeploymentResponse(
command.DeploymentId, "", true, null, DateTimeOffset.UtcNow);
command.DeploymentId, "", secretError is null, secretError, DateTimeOffset.UtcNow,
secretsApplied);
}
catch (Exception ex)
{
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/DeployedSecretsMirror.cs
+34 -13
View File
@@ -17,8 +17,20 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
/// <param name="Applied">Names applied (inserted or updated) this pass.</param>
/// <param name="Removed">Names deleted this pass because they were previously mirrored and are now tombstoned upstream.</param>
/// <param name="Failed">One message per live secret that failed to apply and/or decrypt on this node — never includes secret values.</param>
/// <param name="FailedNames">
/// The secret name behind each <paramref name="Failed"/> entry, in the SAME order (index-for-
/// index — each pass reports at most one failure per row, so the two lists always stay in
/// lockstep). Added for <c>DeploymentManagerActor.HandleDeployArtifacts</c> (Task 10): this
/// mirror reconciles the WHOLE site-wide <c>deployed_secrets</c> table every pass, not only the
/// names a particular deploy just shipped, so a caller that needs to know whether a failure
/// belongs to ITS OWN shipped names — as opposed to some other, previously-deployed secret, or a
/// removal of a now-unreferenced one — cannot recover that from the message strings alone.
/// </param>
public sealed record MirrorResult(
IReadOnlyList<string> Applied, IReadOnlyList<string> Removed, IReadOnlyList<string> Failed)
IReadOnlyList<string> Applied,
IReadOnlyList<string> Removed,
IReadOnlyList<string> Failed,
IReadOnlyList<string> FailedNames)
{
/// <summary>Whether this pass applied or removed anything (used to decide the summary log level).</summary>
public bool Changed => Applied.Count > 0 || Removed.Count > 0;
@@ -209,6 +221,7 @@ public sealed class DeployedSecretsMirror : BackgroundService
List<string> applied = [];
List<string> removed = [];
List<string> failed = [];
List<string> failedNames = [];
foreach (var row in rows)
{
@@ -220,7 +233,7 @@ public sealed class DeployedSecretsMirror : BackgroundService
}
else
{
await ProcessLiveRowAsync(row, mirrored, applied, failed, ct).ConfigureAwait(false);
await ProcessLiveRowAsync(row, mirrored, applied, failed, failedNames, ct).ConfigureAwait(false);
}
}
catch (Exception ex) when (!ct.IsCancellationRequested)
@@ -229,11 +242,12 @@ public sealed class DeployedSecretsMirror : BackgroundService
row.Name,
$"error:{row.Secret.Revision}:{ex.GetType().Name}:{ex.Message}",
$"Secret '{row.Name}' failed to process on this node ({ex.GetType().Name}: {ex.Message})",
failed);
failed,
failedNames);
}
}
var result = new MirrorResult(applied, removed, failed);
var result = new MirrorResult(applied, removed, failed, failedNames);
if (result.Changed)
{
@@ -290,6 +304,7 @@ public sealed class DeployedSecretsMirror : BackgroundService
Dictionary<string, long> mirrored,
List<string> applied,
List<string> failed,
List<string> failedNames,
CancellationToken ct)
{
var name = new SecretName(row.Name);
@@ -323,7 +338,8 @@ public sealed class DeployedSecretsMirror : BackgroundService
$"outofband:{row.Secret.Revision}",
$"Secret '{row.Name}' exists on this node with a newer local value; " +
"the deployed revision was not applied",
failed);
failed,
failedNames);
return; // Do not verify — that would decrypt the operator's own value.
case ApplyOutcome.Unconfirmed:
@@ -332,12 +348,13 @@ public sealed class DeployedSecretsMirror : BackgroundService
$"unconfirmed:{row.Secret.Revision}",
$"Secret '{row.Name}' could not be confirmed applied on this node " +
"after a re-apply attempt",
failed);
failed,
failedNames);
return;
}
}
await VerifyAsync(row, failed, ct).ConfigureAwait(false);
await VerifyAsync(row, failed, failedNames, ct).ConfigureAwait(false);
}
/// <summary>The confirmed outcome of applying one shipped row into this node's store. See the class remarks.</summary>
@@ -419,7 +436,8 @@ public sealed class DeployedSecretsMirror : BackgroundService
/// verify it decrypts on this node. On failure, appends a message to
/// <paramref name="failed"/> and logs with the shared de-spam rule (see the class remarks).
/// </summary>
private async Task VerifyAsync(DeployedSecretRow row, List<string> failed, CancellationToken ct)
private async Task VerifyAsync(
DeployedSecretRow row, List<string> failed, List<string> failedNames, CancellationToken ct)
{
bool resolved;
try
@@ -444,17 +462,20 @@ public sealed class DeployedSecretsMirror : BackgroundService
var message =
$"Secret '{row.Name}' (kek {row.Secret.KekId}) did not decrypt on this node — " +
"the node's master key differs from central's";
ReportFailure(row.Name, $"decrypt:{row.Secret.Revision}:{row.Secret.KekId}", message, failed);
ReportFailure(row.Name, $"decrypt:{row.Secret.Revision}:{row.Secret.KekId}", message, failed, failedNames);
}
/// <summary>
/// Appends <paramref name="message"/> to <paramref name="failed"/> and logs it — Error the
/// first time <paramref name="fingerprint"/> is seen for <paramref name="name"/> since it
/// last succeeded, Debug on every identical repeat.
/// Appends <paramref name="message"/> to <paramref name="failed"/> (and <paramref name="name"/>
/// to <paramref name="failedNames"/>, in lockstep) and logs it — Error the first time
/// <paramref name="fingerprint"/> is seen for <paramref name="name"/> since it last
/// succeeded, Debug on every identical repeat.
/// </summary>
private void ReportFailure(string name, string fingerprint, string message, List<string> failed)
private void ReportFailure(
string name, string fingerprint, string message, List<string> failed, List<string> failedNames)
{
failed.Add(message);
failedNames.Add(name);
if (_lastLoggedFailure.TryGetValue(name, out var previous) && previous == fingerprint)
{