Joseph Doherty
29049c4ef6
Secret-store outage (final review M5): SiteSecretsCollector returns a store read failure as SiteSecretsCollection.StoreError instead of throwing. A deploy now fails only the sites that reference a secret (nothing sent, not stamped, DeploySiteSecrets row with the error), and ComputeCurrentArtifactHashesAsync omits only those sites (they read Out of date) instead of the exception aborting the whole sweep and the fleet-wide pending-updates check. Removal-only deploys: the site reply gains SecretsRemoved (proto field 7, names only) - the deployed_secrets rows the deploy soft-removed. Central carries it on SiteArtifactResult and writes the DeploySiteSecrets row with a removed list whenever it is non-empty, so dropping a site's last secret: reference is audited centrally rather than only in site logs.
415 lines
17 KiB
C#
415 lines
17 KiB
C#
using Akka.Actor;
|
|
using Akka.TestKit;
|
|
using Akka.TestKit.Xunit2;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
|
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
|
|
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
|
|
|
/// <summary>
|
|
/// Task 10, site-secrets artifact deploy: <see cref="DeploymentManagerActor.HandleDeployArtifacts"/>
|
|
/// writes shipped secrets to <c>deployed_secrets</c>, runs <see cref="DeployedSecretsMirror.ReconcileAsync"/>
|
|
/// synchronously, and answers with <see cref="ArtifactDeploymentResponse.SecretsApplied"/> or a failure.
|
|
/// Follows the shape of the existing artifact-apply tests in <c>DeploymentManagerActorTests</c>.
|
|
/// </summary>
|
|
public class DeploymentManagerActorArtifactSecretsTests : TestKit, IDisposable
|
|
{
|
|
private readonly SiteStorageService _storage;
|
|
private readonly ScriptCompilationService _compilationService;
|
|
private readonly SharedScriptLibrary _sharedScriptLibrary;
|
|
private readonly TestLocalDb _localDb;
|
|
|
|
public DeploymentManagerActorArtifactSecretsTests()
|
|
{
|
|
_localDb = TestLocalDb.CreateTemp("dm-artifact-secrets-test");
|
|
_storage = new SiteStorageService(
|
|
_localDb.Db,
|
|
NullLogger<SiteStorageService>.Instance);
|
|
_storage.InitializeAsync().GetAwaiter().GetResult();
|
|
_compilationService = new ScriptCompilationService(
|
|
NullLogger<ScriptCompilationService>.Instance);
|
|
_sharedScriptLibrary = new SharedScriptLibrary(
|
|
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
|
}
|
|
|
|
void IDisposable.Dispose()
|
|
{
|
|
Shutdown();
|
|
var path = _localDb.Path;
|
|
_localDb.Dispose();
|
|
TestLocalDb.DeleteFiles(path);
|
|
}
|
|
|
|
private IActorRef CreateDeploymentManager(DeployedSecretsMirror? secretsMirror = null, IActorRef? dclManager = null)
|
|
{
|
|
var options = new SiteRuntimeOptions();
|
|
// Positional, not named, past the constructor's first six parameters: a named
|
|
// argument for a trailing optional parameter (secretsMirror) that SKIPS other,
|
|
// still-optional parameters in between is rejected by the compiler inside an
|
|
// expression-tree lambda (Props.Create's Expression<Func<T>> overload) with
|
|
// CS9307 "named argument specification out of position".
|
|
return ActorOf(Props.Create(() => new DeploymentManagerActor(
|
|
_storage,
|
|
_compilationService,
|
|
_sharedScriptLibrary,
|
|
null, // streamManager: no stream manager in tests
|
|
options,
|
|
NullLogger<DeploymentManagerActor>.Instance,
|
|
dclManager,
|
|
null, // healthCollector
|
|
null, // serviceProvider
|
|
null, // loggerFactory
|
|
null, // configFetcher
|
|
null, // startupLoadRetryInterval
|
|
null, // configLoader
|
|
null, // externalSystemCache
|
|
null, // scriptScheduler
|
|
null, // connectionDefinitionLoader
|
|
null, // deployedConfigLoader
|
|
secretsMirror)));
|
|
}
|
|
|
|
private DeployedSecretsMirror CreateMirror(FakeSecretStore store, FakeSecretResolver resolver)
|
|
{
|
|
var options = Options.Create(new SiteRuntimeOptions
|
|
{
|
|
DeployedSecretsMirrorInterval = TimeSpan.FromSeconds(15),
|
|
});
|
|
return new DeployedSecretsMirror(
|
|
_storage,
|
|
store,
|
|
resolver,
|
|
invalidator: null,
|
|
options,
|
|
NullLogger<DeployedSecretsMirror>.Instance,
|
|
TimeProvider.System);
|
|
}
|
|
|
|
private static DeployArtifactsCommand ArtifactsCommand(
|
|
string deploymentId, IReadOnlyList<SecretArtifact>? secrets) => new(
|
|
DeploymentId: deploymentId,
|
|
SharedScripts: null,
|
|
ExternalSystems: null,
|
|
DatabaseConnections: null,
|
|
NotificationLists: null,
|
|
DataConnections: null,
|
|
SmtpConfigurations: null,
|
|
Timestamp: DateTimeOffset.UtcNow,
|
|
Secrets: secrets);
|
|
|
|
private static SecretArtifact Art(string name, long revision, string kek = "kek-1") => new(
|
|
Name: name,
|
|
ContentType: "Text",
|
|
Ciphertext: [1, 2, 3, 4, 5],
|
|
Nonce: [10, 11, 12],
|
|
Tag: [20, 21, 22, 23],
|
|
WrappedDek: [30, 31, 32, 33, 34, 35],
|
|
WrapNonce: [40, 41, 42],
|
|
WrapTag: [50, 51, 52, 53],
|
|
KekId: kek,
|
|
Revision: revision,
|
|
CreatedUtc: new DateTimeOffset(2026, 9, 28, 0, 0, 0, TimeSpan.Zero),
|
|
UpdatedUtc: new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero),
|
|
Description: "test secret");
|
|
|
|
[Fact]
|
|
public async Task Secrets_Null_LeavesTableUntouched_AndSecretsAppliedNull()
|
|
{
|
|
var store = new FakeSecretStore();
|
|
var resolver = new FakeSecretResolver();
|
|
var mirror = CreateMirror(store, resolver);
|
|
var manager = CreateDeploymentManager(mirror);
|
|
|
|
manager.Tell(ArtifactsCommand("dep-null", secrets: null));
|
|
|
|
var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.True(response.Success);
|
|
Assert.Null(response.SecretsApplied);
|
|
|
|
var rows = await _storage.GetDeployedSecretsAsync();
|
|
Assert.Empty(rows);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Secrets_Present_FullSetWrite_SyncReconcile_SecretsAppliedEqualsShippedCount()
|
|
{
|
|
var store = new FakeSecretStore();
|
|
var resolver = new FakeSecretResolver();
|
|
var mirror = CreateMirror(store, resolver);
|
|
var manager = CreateDeploymentManager(mirror);
|
|
|
|
manager.Tell(ArtifactsCommand("dep-2secrets", secrets: [Art("a", 1), Art("b", 1)]));
|
|
|
|
var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.True(response.Success);
|
|
Assert.Equal(2, response.SecretsApplied);
|
|
Assert.Null(response.ErrorMessage);
|
|
|
|
// Reconcile ran synchronously before the reply — both secrets already applied.
|
|
Assert.True(store.Contains("a"));
|
|
Assert.True(store.Contains("b"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Secrets_Empty_WithMirror_SecretsAppliedZero_Success()
|
|
{
|
|
var store = new FakeSecretStore();
|
|
var resolver = new FakeSecretResolver();
|
|
var mirror = CreateMirror(store, resolver);
|
|
var manager = CreateDeploymentManager(mirror);
|
|
|
|
manager.Tell(ArtifactsCommand("dep-empty", secrets: []));
|
|
|
|
var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.True(response.Success);
|
|
Assert.Equal(0, response.SecretsApplied);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Secrets_Dropped_ReplyNamesTheRetiredSecrets()
|
|
{
|
|
var store = new FakeSecretStore();
|
|
var resolver = new FakeSecretResolver();
|
|
var mirror = CreateMirror(store, resolver);
|
|
var manager = CreateDeploymentManager(mirror);
|
|
|
|
manager.Tell(ArtifactsCommand("dep-1", secrets: [Art("a", 1), Art("b", 1)]));
|
|
var first = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.True(first.Success);
|
|
Assert.Null(first.SecretsRemoved);
|
|
|
|
// The next deploy no longer references "b" (a removal-only change for that name):
|
|
// the reply names it so central can audit the removal.
|
|
manager.Tell(ArtifactsCommand("dep-2", secrets: [Art("a", 1)]));
|
|
var second = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.True(second.Success);
|
|
Assert.Equal(["b"], second.SecretsRemoved!);
|
|
|
|
// Dropping the last reference retires "a" the same way.
|
|
manager.Tell(ArtifactsCommand("dep-3", secrets: []));
|
|
var third = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.True(third.Success);
|
|
Assert.Equal(0, third.SecretsApplied);
|
|
Assert.Equal(["a"], third.SecretsRemoved!);
|
|
await Task.CompletedTask;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Secrets_MirrorFailsOne_SuccessFalse_KekIdInMessage_SecretsAppliedExcludesFailure()
|
|
{
|
|
var store = new FakeSecretStore();
|
|
var resolver = new FakeSecretResolver();
|
|
resolver.ThrowFor("bad");
|
|
var mirror = CreateMirror(store, resolver);
|
|
var manager = CreateDeploymentManager(mirror);
|
|
|
|
manager.Tell(ArtifactsCommand(
|
|
"dep-onefails", secrets: [Art("good", 1), Art("bad", 1, kek: "kek-central")]));
|
|
|
|
var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.False(response.Success);
|
|
Assert.NotNull(response.ErrorMessage);
|
|
Assert.Contains("kek-central", response.ErrorMessage);
|
|
Assert.Contains("'bad'", response.ErrorMessage);
|
|
Assert.Equal(1, response.SecretsApplied); // "good" applied+verified; "bad" failed
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Secrets_NonEmpty_NoMirrorRegistered_FailsDeploy()
|
|
{
|
|
var manager = CreateDeploymentManager(secretsMirror: null);
|
|
|
|
manager.Tell(ArtifactsCommand("dep-nomirror", secrets: [Art("a", 1)]));
|
|
|
|
var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.False(response.Success);
|
|
Assert.Equal("Secret deployment is not enabled on this site node", response.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Secrets_Empty_NoMirrorRegistered_TreatedAsZeroSuccess()
|
|
{
|
|
var manager = CreateDeploymentManager(secretsMirror: null);
|
|
|
|
manager.Tell(ArtifactsCommand("dep-nomirror-empty", secrets: []));
|
|
|
|
var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.True(response.Success);
|
|
Assert.Equal(0, response.SecretsApplied);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Secrets_RemovalFailure_DoesNotFailDeploy_AndExcludedFromErrorMessage()
|
|
{
|
|
// A row that was previously mirrored (ledgered) but is now tombstoned upstream, and
|
|
// whose removal fails on this node's store — a failure UNRELATED to the names shipped
|
|
// by THIS deploy. Per ruling: log at Warning, never surface in ErrorMessage, never fail
|
|
// the deploy, never affect SecretsApplied (which only counts SHIPPED names).
|
|
var store = new FakeSecretStore();
|
|
var resolver = new FakeSecretResolver();
|
|
var mirror = CreateMirror(store, resolver);
|
|
var manager = CreateDeploymentManager(mirror);
|
|
|
|
// First deploy ships + ledgers "gone".
|
|
manager.Tell(ArtifactsCommand("dep-r1", secrets: [Art("gone", 1)]));
|
|
var r1 = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.True(r1.Success);
|
|
Assert.True(store.Contains("gone"));
|
|
|
|
// Make the store's delete fail on the next pass, then ship a deploy that no longer
|
|
// references "gone" (and ships an unrelated, healthy secret).
|
|
store.ThrowOnDelete = true;
|
|
manager.Tell(ArtifactsCommand("dep-r2", secrets: [Art("kept", 1)]));
|
|
var r2 = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
|
|
Assert.True(r2.Success);
|
|
Assert.Equal(1, r2.SecretsApplied); // only "kept" is shipped by this deploy
|
|
Assert.Null(r2.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void DataConnections_PushedToDcl_OnlyAfterShippedSecretsAreMirrored()
|
|
{
|
|
// M2 (final review): a connection whose config references a secret shipped by THIS
|
|
// deploy must find it on its first connect, so the DCL create must not reach the
|
|
// DCL manager before the secret is in this node's store.
|
|
var store = new FakeSecretStore();
|
|
var resolver = new FakeSecretResolver();
|
|
var mirror = CreateMirror(store, resolver);
|
|
var dcl = CreateTestProbe();
|
|
bool? secretPresentWhenDclSawCreate = null;
|
|
dcl.SetAutoPilot(new DelegateAutoPilot((_, msg) =>
|
|
{
|
|
if (msg is ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection.CreateConnectionCommand)
|
|
{
|
|
secretPresentWhenDclSawCreate ??= store.Contains("gw-key");
|
|
}
|
|
|
|
return AutoPilot.KeepRunning;
|
|
}));
|
|
var manager = CreateDeploymentManager(mirror, dcl.Ref);
|
|
|
|
var command = ArtifactsCommand("dep-order", secrets: [Art("gw-key", 1)]) with
|
|
{
|
|
DataConnections = [new DataConnectionArtifact("Gw", "OpcUa", "{}", null, 3)],
|
|
};
|
|
manager.Tell(command);
|
|
|
|
var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.True(response.Success);
|
|
Assert.Equal(1, response.SecretsApplied);
|
|
AwaitCondition(() => secretPresentWhenDclSawCreate.HasValue, TimeSpan.FromSeconds(10));
|
|
Assert.True(secretPresentWhenDclSawCreate);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DataConnections_StillStoredAndPushed_WhenSecretsStepFails()
|
|
{
|
|
// The reorder must not lose the connection push when the secrets step fails.
|
|
var dcl = CreateTestProbe();
|
|
var manager = CreateDeploymentManager(secretsMirror: null, dclManager: dcl.Ref);
|
|
|
|
var command = ArtifactsCommand("dep-order-fail", secrets: [Art("gw-key", 1)]) with
|
|
{
|
|
DataConnections = [new DataConnectionArtifact("Gw", "OpcUa", "{}", null, 3)],
|
|
};
|
|
manager.Tell(command);
|
|
|
|
var response = ExpectMsg<ArtifactDeploymentResponse>(TimeSpan.FromSeconds(10));
|
|
Assert.False(response.Success);
|
|
dcl.ExpectMsg<ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection.CreateConnectionCommand>(
|
|
TimeSpan.FromSeconds(10));
|
|
Assert.Contains(await _storage.GetAllDataConnectionDefinitionsAsync(), d => d.Name == "Gw");
|
|
}
|
|
|
|
// ── Fakes (mirrors DeployedSecretsMirrorTests' fakes; kept local/minimal per-task) ──
|
|
|
|
private sealed class FakeSecretStore : ISecretStore
|
|
{
|
|
private readonly Dictionary<string, StoredSecret> _rows = new(StringComparer.Ordinal);
|
|
|
|
public bool ThrowOnDelete { get; set; }
|
|
|
|
public bool Contains(string name) => _rows.TryGetValue(name, out var row) && !row.IsDeleted;
|
|
|
|
public Task<StoredSecret?> GetAsync(SecretName name, CancellationToken ct) =>
|
|
Task.FromResult(_rows.TryGetValue(name.Value, out var row) ? row : null);
|
|
|
|
public Task UpsertAsync(StoredSecret row, CancellationToken ct)
|
|
{
|
|
_rows[row.Name.Value] = row;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public Task<bool> DeleteAsync(SecretName name, string? actor, CancellationToken ct)
|
|
{
|
|
if (ThrowOnDelete)
|
|
{
|
|
throw new InvalidOperationException("simulated delete fault");
|
|
}
|
|
|
|
if (!_rows.TryGetValue(name.Value, out var existing) || existing.IsDeleted)
|
|
{
|
|
return Task.FromResult(false);
|
|
}
|
|
|
|
_rows[name.Value] = existing with
|
|
{
|
|
IsDeleted = true,
|
|
DeletedUtc = DateTimeOffset.UtcNow,
|
|
UpdatedUtc = DateTimeOffset.UtcNow,
|
|
Revision = existing.Revision + 1,
|
|
UpdatedBy = actor,
|
|
};
|
|
return Task.FromResult(true);
|
|
}
|
|
|
|
public Task<IReadOnlyList<SecretMetadata>> ListAsync(bool includeDeleted, CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<SecretMetadata>>([]);
|
|
|
|
public Task<IReadOnlyList<SecretManifestEntry>> GetManifestAsync(CancellationToken ct) =>
|
|
Task.FromResult<IReadOnlyList<SecretManifestEntry>>([]);
|
|
|
|
public Task ApplyReplicatedAsync(StoredSecret row, CancellationToken ct)
|
|
{
|
|
if (_rows.TryGetValue(row.Name.Value, out var existing)
|
|
&& !SecretLastWriterWins.IsNewer(row.UpdatedUtc, row.Revision, existing.UpdatedUtc, existing.Revision))
|
|
{
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
_rows[row.Name.Value] = row;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public Task<bool> ApplyRewrapAsync(
|
|
StoredSecret rewrappedRow, byte[] expectedCurrentWrappedDek, CancellationToken ct) =>
|
|
Task.FromResult(false);
|
|
}
|
|
|
|
private sealed class FakeSecretResolver : ISecretResolver
|
|
{
|
|
private readonly HashSet<string> _throwing = new(StringComparer.Ordinal);
|
|
|
|
public void ThrowFor(string name) => _throwing.Add(name);
|
|
|
|
public Task<string?> GetAsync(SecretName name, CancellationToken ct)
|
|
{
|
|
if (_throwing.Contains(name.Value))
|
|
{
|
|
throw new InvalidOperationException("simulated decrypt failure");
|
|
}
|
|
|
|
return Task.FromResult<string?>("decrypted-value");
|
|
}
|
|
}
|
|
}
|