Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerMediumFindingsTests.cs
T
Joseph Doherty 7c1268d85c feat(dcl+siteruntime): reconfigure a LIVE data connection in place — register row 97
`DataConnectionManagerActor.HandleCreateConnection` is create-only (warns and
returns for a name it already holds), so `EnsureDclConnection`'s documented
"a changed config re-issues the create so the DCL adopts it" was never true on
ANY node: a corrected endpoint/credential/backup/failover-count took effect only
when the connection was next built from scratch (node restart, singleton
failover, remove-then-re-add), while the DM's hash cache recorded it as applied.
Rows 89/90 made that honest with a Warning; this closes it.

New `ReconfigureConnectionCommand` (Commons, same shape as the create):

- `DeploymentManagerActor.EnsureDclConnection` now has three outcomes off its one
  per-name hash slot — unchanged -> nothing at all, new -> create, CHANGED ->
  reconfigure — and logs "Reconfigured DCL connection {Connection}
  (protocol={Protocol}) in place" at Information in place of the old Warning. The
  winner-resolution / arbitration / tombstone logic is untouched; only what
  happens after a change is detected changed.
- `DataConnectionManagerActor` FORWARDS the command to the owning connection
  actor, falling back to a create for a name it does not hold, so the command is
  safe whatever the manager's state.
- `DataConnectionActor` handles it in ALL THREE states and never stashes it — a
  connection pointed at a wrong or dead endpoint lives in Connecting/Reconnecting,
  which is exactly when a correction arrives; stashed it would wait on the broken
  configuration forever. Handling replaces protocol/primary/backup/failover-count,
  re-arms endpoint selection to Primary (the new config's primary is the intent,
  whatever the old config had failed over to), and calls the new shared
  `SwapAdapter` helper — now used by both failover sites too, so the swap exists
  once: unhook + fire-and-forget dispose -> _factory.Create -> rewire -> bump
  _adapterGeneration LAST. `BecomeReconnecting(afterReconfigure: true)` then
  reconnects immediately (StartSingleTimer replaces any armed reconnect, so no
  racing attempts) with the unstable-disconnect accounting and failover check
  suppressed — the link was dropped on purpose, and counting it as an unstable
  cycle could trip a failover onto the NEW backup before the new primary had been
  tried once. ReSubscribeAll/ReSubscribeAllAlarms then re-issue everything from
  the untouched subscription registries. Idempotent: a reconfigure carrying the
  configuration already in force is a no-op.

Stop-and-recreate was rejected as destructive: subscriptions live inside
DataConnectionActor and InstanceActor cancels its subscribe-retry timer on first
success, so a rebuilt actor would have no subscribers and nothing would ever
re-subscribe.

Tests: new DataConnectionReconfigureTests (7; 5 fail / 2 pass against a
behavioural revert). DclConnectionArbitrationTests gains two row-97 tests and its
CDC-arrival test now asserts the second push is a reconfigure; its ping-pong
guard also asserts zero reconfigures, since a flip would now show up there.
SiteRuntime-010 renamed to ..._ReissuesReconfigureCommand. 3 fail against a
reverted EnsureDclConnection. DataConnectionLayer.Tests 327/327,
SiteRuntime.Tests 643/643, solution build 0 warnings.

Docs: Component-DataConnectionLayer.md gains an "In-place reconfiguration"
section under Endpoint Redundancy; Component-SiteRuntime.md, and the
docs/components/ mirrors, updated; the arbitration decision memo gains an
appended §9 postscript resolving its residual 1 (residuals 2 and 3 stay open).
2026-08-17 06:19:13 -04:00

278 lines
12 KiB
C#

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
using ZB.MOM.WW.ScadaBridge.TestSupport;
using System.Text.Json;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
/// <summary>
/// Regression tests for the Medium-severity DeploymentManagerActor findings:
/// SiteRuntime-005 (Success reported before persistence completes) and
/// SiteRuntime-008 (blocking shared-script load on the actor thread).
/// </summary>
public class DeploymentManagerMediumFindingsTests : TestKit, IDisposable
{
private readonly ScriptCompilationService _compilationService;
private readonly SharedScriptLibrary _sharedScriptLibrary;
private readonly TestLocalDb _localDb;
public DeploymentManagerMediumFindingsTests()
{
_localDb = TestLocalDb.CreateTemp("dm-medium-test");
_compilationService = new ScriptCompilationService(
NullLogger<ScriptCompilationService>.Instance);
_sharedScriptLibrary = new SharedScriptLibrary(
_compilationService, NullLogger<SharedScriptLibrary>.Instance);
}
void IDisposable.Dispose()
{
// TestKit teardown first, so no in-flight actor can reach a disposed ILocalDb;
// then dispose the database before deleting — the master connection anchors the WAL.
Shutdown();
var path = _localDb.Path;
_localDb.Dispose();
TestLocalDb.DeleteFiles(path);
}
private SiteStorageService NewStorage(ILocalDb localDb)
=> new(localDb, NullLogger<SiteStorageService>.Instance);
private IActorRef CreateDeploymentManager(SiteStorageService storage, IActorRef? dclManager = null)
{
return ActorOf(Props.Create(() => new DeploymentManagerActor(
storage,
_compilationService,
_sharedScriptLibrary,
null,
new SiteRuntimeOptions(),
NullLogger<DeploymentManagerActor>.Instance,
dclManager,
null,
null,
null)));
}
private static string MakeConfigJsonWithConnection(
string instanceName, string endpoint, int failoverRetryCount)
{
var config = new FlattenedConfiguration
{
InstanceUniqueName = instanceName,
Attributes =
[
new ResolvedAttribute { CanonicalName = "TestAttr", Value = "1", DataType = "Int32" }
],
Connections = new Dictionary<string, ConnectionConfig>
{
["Conn1"] = new ConnectionConfig
{
Protocol = "Custom",
ConfigurationJson = $"{{\"endpoint\":\"{endpoint}\"}}",
FailoverRetryCount = failoverRetryCount
}
}
};
return JsonSerializer.Serialize(config);
}
private static string MakeConfigJson(string instanceName)
{
var config = new FlattenedConfiguration
{
InstanceUniqueName = instanceName,
Attributes =
[
new ResolvedAttribute { CanonicalName = "TestAttr", Value = "1", DataType = "Int32" }
]
};
return JsonSerializer.Serialize(config);
}
/// <summary>
/// SiteRuntime-005: when SQLite persistence of the deployed config fails, the
/// Deployment Manager must report <see cref="DeploymentStatus.Failed"/> to central,
/// not <see cref="DeploymentStatus.Success"/>. Reporting Success on a persistence
/// failure silently loses the deployment on the next restart/failover.
/// </summary>
[Fact]
public async Task Deploy_PersistenceFailure_ReportsFailedNotSuccess()
{
// A storage over a database whose site tables were never created makes every
// storage operation throw ("no such table"), so StoreDeployedConfigAsync fails.
//
// This replaces the old "connection string pointing at an unwritable path" trick:
// the service now takes an ILocalDb rather than a connection string, and LocalDb
// opens its file eagerly in its own constructor — so an unopenable path fails
// while building the fixture, before there is a storage to hand the actor at all.
// Skipping InitializeAsync is the equivalent lever on the new seam, and fails the
// same way the old one did: reads AND writes both throw.
var uninitialized = TestLocalDb.CreateTemp("dm-medium-persist-fail");
try
{
var storage = NewStorage(uninitialized.Db);
var actor = CreateDeploymentManager(storage);
await Task.Delay(500); // empty startup
actor.Tell(new DeployInstanceCommand(
"dep-fail", "FailPump", "h1", MakeConfigJson("FailPump"), "admin", DateTimeOffset.UtcNow));
var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(10));
Assert.Equal("FailPump", response.InstanceUniqueName);
Assert.Equal(DeploymentStatus.Failed, response.Status);
Assert.False(string.IsNullOrEmpty(response.ErrorMessage));
}
finally
{
var path = uninitialized.Path;
uninitialized.Dispose();
TestLocalDb.DeleteFiles(path);
}
}
/// <summary>
/// SiteRuntime-005: a successful deployment must still report
/// <see cref="DeploymentStatus.Success"/>, and only after the config row is
/// committed to SQLite (so a restart re-creates the instance).
/// </summary>
[Fact]
public async Task Deploy_Success_ReportsSuccessAndPersistsConfig()
{
var storage = NewStorage(_localDb.Db);
await storage.InitializeAsync();
var actor = CreateDeploymentManager(storage);
await Task.Delay(500);
actor.Tell(new DeployInstanceCommand(
"dep-ok", "OkPump", "h1", MakeConfigJson("OkPump"), "admin", DateTimeOffset.UtcNow));
var response = ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(10));
Assert.Equal(DeploymentStatus.Success, response.Status);
// By the time Success is reported, the config must be durable.
var configs = await storage.GetAllDeployedConfigsAsync();
Assert.Contains(configs, c => c.InstanceUniqueName == "OkPump");
}
/// <summary>
/// SiteRuntime-010: when a redeployment changes a connection's configuration
/// (here the failover retry count and endpoint), the Deployment Manager must push the new
/// configuration to the DCL rather than leaving it serving the stale one.
///
/// <para>
/// <b>Register row 97 changed WHICH command that is.</b> It used to be a re-issued
/// <c>CreateConnectionCommand</c> — which the create-only manager silently dropped for a name
/// it already held, so this test proved a message was sent and nothing more. It is now a
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection.ReconfigureConnectionCommand"/>,
/// which the live connection actor actually applies in place.
/// </para>
/// </summary>
[Fact]
public async Task EnsureDclConnections_ConnectionConfigChanged_ReissuesReconfigureCommand()
{
var storage = NewStorage(_localDb.Db);
await storage.InitializeAsync();
var dcl = CreateTestProbe();
var actor = CreateDeploymentManager(storage, dcl.Ref);
await Task.Delay(500);
// Initial deploy with one connection.
actor.Tell(new DeployInstanceCommand(
"dep-c1", "ConnPump", "h1",
MakeConfigJsonWithConnection("ConnPump", "opc.tcp://host-a:4840", 3),
"admin", DateTimeOffset.UtcNow));
var firstCreate = dcl.ExpectMsg<ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection.CreateConnectionCommand>(
TimeSpan.FromSeconds(5));
Assert.Equal("Conn1", firstCreate.ConnectionName);
Assert.Equal(3, firstCreate.FailoverRetryCount);
ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
await Task.Delay(500);
// Redeploy with a CHANGED connection configuration.
actor.Tell(new DeployInstanceCommand(
"dep-c2", "ConnPump", "h2",
MakeConfigJsonWithConnection("ConnPump", "opc.tcp://host-b:4840", 7),
"admin", DateTimeOffset.UtcNow));
// The DCL must receive a RECONFIGURE reflecting the new config — not a second create,
// which the create-only manager would drop for a name it already holds.
var reconfigure = dcl.ExpectMsg<ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection.ReconfigureConnectionCommand>(
TimeSpan.FromSeconds(10));
Assert.Equal("Conn1", reconfigure.ConnectionName);
Assert.Equal(7, reconfigure.FailoverRetryCount);
}
/// <summary>
/// SiteRuntime-010: an unchanged connection configuration must still be skipped —
/// re-sending an identical create command on every deploy is wasteful.
/// </summary>
[Fact]
public async Task EnsureDclConnections_UnchangedConfig_DoesNotReissueCreateCommand()
{
var storage = NewStorage(_localDb.Db);
await storage.InitializeAsync();
var dcl = CreateTestProbe();
var actor = CreateDeploymentManager(storage, dcl.Ref);
await Task.Delay(500);
var json = MakeConfigJsonWithConnection("StablePump", "opc.tcp://host-a:4840", 3);
actor.Tell(new DeployInstanceCommand(
"dep-s1", "StablePump", "h1", json, "admin", DateTimeOffset.UtcNow));
dcl.ExpectMsg<ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection.CreateConnectionCommand>(
TimeSpan.FromSeconds(5));
ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(5));
await Task.Delay(500);
// Redeploy with the IDENTICAL connection configuration.
actor.Tell(new DeployInstanceCommand(
"dep-s2", "StablePump", "h2", json, "admin", DateTimeOffset.UtcNow));
ExpectMsg<DeploymentStatusResponse>(TimeSpan.FromSeconds(10));
// No further create command for an unchanged connection.
dcl.ExpectNoMsg(TimeSpan.FromMilliseconds(800));
}
/// <summary>
/// SiteRuntime-008: startup must not block the Deployment Manager mailbox on a
/// synchronous shared-script load. With shared scripts present, the actor must
/// still load deployed configs, create Instance Actors, and remain responsive.
/// </summary>
[Fact]
public async Task Startup_WithSharedScripts_LoadsConfigsAndStaysResponsive()
{
var storage = NewStorage(_localDb.Db);
await storage.InitializeAsync();
// Several shared scripts to compile during startup.
for (var i = 0; i < 5; i++)
{
await storage.StoreSharedScriptAsync(
$"Shared{i}", "return 1 + 1;", null, null);
}
await storage.StoreDeployedConfigAsync(
"StartupPump", MakeConfigJson("StartupPump"), "d1", "h1", true);
var actor = CreateDeploymentManager(storage);
await Task.Delay(2000);
// The instance loaded at startup must be operable — proves startup completed
// and the actor processed messages after the shared-script load.
actor.Tell(new DeploymentStateQueryRequest("corr-1", "StartupPump", DateTimeOffset.UtcNow));
var response = ExpectMsg<DeploymentStateQueryResponse>(TimeSpan.FromSeconds(5));
Assert.True(response.IsDeployed);
}
}