diff --git a/alog.md b/alog.md index c8f0921b..8a265315 100644 --- a/alog.md +++ b/alog.md @@ -108,7 +108,7 @@ Single wide table, polymorphic by `Channel` + `Kind` discriminators, JSON payloa | `EventId` | `uniqueidentifier` PK | Generated where the event originates (site or central). Idempotency key. | | `OccurredAtUtc` | `datetime2` | When the event happened (call returned, retry attempted, etc.). | | `IngestedAtUtc` | `datetime2` | When central persisted the row (lags `OccurredAtUtc` for site-originated rows). | -| `Channel` | `varchar(32)` | `ApiOutbound` \| `DbOutbound` \| `Notification` \| `ApiInbound`. | +| `Channel` | `varchar(32)` | `ApiOutbound` \| `DbOutbound` \| `Notification` \| `ApiInbound` \| `SecuredWrite` \| `Cluster`. The last two are not script trust-boundary crossings: `SecuredWrite` records the two-person write lifecycle, and `Cluster` records operator-initiated topology actions (admin-triggered manual failover, decision 2026-07-22). | | `Kind` | `varchar(32)` | Event kind discriminator (see kinds list below). | | `CorrelationId` | `uniqueidentifier` NULL | Ties multi-event operations together. `TrackedOperationId` for cached calls, `NotificationId` for notifications, request-id for inbound API. NULL for sync one-shot calls. | | `SourceSiteId` | `varchar(64)` NULL | NULL for central-originated events (inbound API, central notification dispatch). | @@ -135,7 +135,7 @@ Single wide table, polymorphic by `Channel` + `Kind` discriminators, JSON payloa - `IX_AuditLog_Target_Occurred (Target, OccurredAtUtc)` — "what did we send to system X." - Partitioning by month on `OccurredAtUtc` from day one (purge becomes a partition switch instead of a delete storm). -**`Kind` values (flat — 10 discriminators across all channels):** +**`Kind` values (flat — 17 discriminators across all channels; pinned by `AuditEnumTests`):** | Kind | Fires when | |---|---| @@ -149,6 +149,13 @@ Single wide table, polymorphic by `Channel` + `Kind` discriminators, JSON payloa | `InboundAuthFailure` | An inbound API request was rejected at the auth boundary (bad/missing key). One row, `Status=Failed`, `HttpStatus=401`. | | `CachedSubmit` | Script-side enqueue of a cached call (`ExternalSystem.CachedCall` / `Database.CachedWrite`); first row in the cached-call lifecycle, written to site SQLite before any forward attempt. | | `CachedResolve` | Terminal row for a cached operation — `Status` = `Delivered` / `Failed` / `Parked` / `Discarded`. | +| `SecuredWriteSubmit` | An Operator submitted a two-person secured write; row written after the `PendingSecuredWrite` is persisted so it carries the store-assigned id as `CorrelationId`. | +| `SecuredWriteApprove` | A distinct Verifier approved a pending secured write (no self-approval; enforced server-side). | +| `SecuredWriteReject` | A Verifier rejected a pending secured write. | +| `SecuredWriteExecute` | An approved secured write was relayed to the site MxGateway connection. | +| `SecuredWriteExpire` | A `Pending` secured write aged past its server-side TTL and was transitioned to `Expired` without executing — emitted by the system (no verifier). | +| `ReconciliationAbandoned` | A reconciliation pull row failed to insert up to the permanent-abandon threshold and central advanced its cursor past it; one synthetic row so the loss is queryable in the Audit Log itself. | +| `ManualFailover` | An administrator triggered a manual failover of the central pair from the Health page; one row per invocation, written BEFORE the graceful `Cluster.Leave` is issued. `Target` = the leaving node's address. | ### Site: `AuditLog` (SQLite) diff --git a/src/ZB.MOM.WW.ScadaBridge.CentralUI/Services/IManualFailoverService.cs b/src/ZB.MOM.WW.ScadaBridge.CentralUI/Services/IManualFailoverService.cs new file mode 100644 index 00000000..16de3ffc --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.CentralUI/Services/IManualFailoverService.cs @@ -0,0 +1,24 @@ +namespace ZB.MOM.WW.ScadaBridge.CentralUI.Services; + +/// +/// Admin-triggered manual failover of the central pair. Declared here — and in terms of +/// plain strings — so CentralUI stays Akka-free; the Akka implementation lives in the Host +/// (AkkaManualFailoverService) and is registered only in the Central branch. +/// +public interface IManualFailoverService +{ + /// + /// Gracefully fails over the central cluster: the current active (oldest Up) member + /// leaves, restarts via its supervisor, and rejoins as standby. The Leave is graceful, + /// never a Down, so cluster singletons hand over instead of being killed. + /// + /// The caller is usually connected THROUGH the node being failed over (Traefik routes + /// the UI to the active node), so the calling Blazor circuit should expect to drop and + /// reconnect against the new active node. + /// + /// + /// Authenticated user name, recorded on the audit row. + /// The address acted on, or null when there is no peer to fail over to + /// (failing over a lone node would be an outage, not a failover). + Task FailOverCentralAsync(string actor); +} diff --git a/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Enums/AuditChannel.cs b/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Enums/AuditChannel.cs index 53b0b4b4..3e643b62 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Enums/AuditChannel.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Enums/AuditChannel.cs @@ -3,7 +3,8 @@ namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; /// /// Top-level Audit Log channel — the trust boundary the audited action crosses. /// One of: outbound API call, outbound DB write, notification send/deliver, inbound API request, -/// or a two-person ("secured") write through its submit/approve/reject/execute lifecycle. +/// a two-person ("secured") write through its submit/approve/reject/execute lifecycle, or an +/// operator-initiated cluster topology action. /// public enum AuditChannel { @@ -11,5 +12,14 @@ public enum AuditChannel DbOutbound, Notification, ApiInbound, - SecuredWrite + SecuredWrite, + + /// + /// An operator-initiated change to cluster topology — currently only the admin-triggered + /// manual failover of the central pair. Distinct from the script trust boundary the other + /// channels describe: nothing here crosses into user script, but a human deliberately + /// restarted the active node, which is exactly the kind of act an audit log exists to + /// attribute. (decision 2026-07-22) + /// + Cluster } diff --git a/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Enums/AuditKind.cs b/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Enums/AuditKind.cs index 67da50ee..e5ed83d0 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Enums/AuditKind.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Enums/AuditKind.cs @@ -40,5 +40,14 @@ public enum AuditKind /// EventId, source site, and final error) so the loss is queryable in /// the Audit Log itself, not only in a rotating log file. /// - ReconciliationAbandoned + ReconciliationAbandoned, + + /// + /// An administrator triggered a manual failover of the central pair from the Health page: + /// the active (oldest Up) node was asked to leave the cluster gracefully so its singletons + /// hand over and the standby takes over. One row per invocation, written BEFORE the Leave + /// is issued (the acting node is usually not the one that goes away, but the audit must + /// survive either outcome). (decision 2026-07-22) + /// + ManualFailover } diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Health/AkkaManualFailoverService.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Health/AkkaManualFailoverService.cs new file mode 100644 index 00000000..ca7a03ce --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Health/AkkaManualFailoverService.cs @@ -0,0 +1,140 @@ +using System.Text.Json; +using Akka.Actor; +using Akka.Cluster; +using Microsoft.Extensions.Logging; +using ZB.MOM.WW.ScadaBridge.CentralUI.Services; +using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; +using ZB.MOM.WW.ScadaBridge.Host.Actors; + +namespace ZB.MOM.WW.ScadaBridge.Host.Health; + +/// +/// Production backed by the running Akka.NET cluster. +/// Registered only in the Central-role branch of Program.cs. +/// +/// Leave, never Down. The target is asked to leave gracefully, so +/// ClusterSingletonManager hands its singletons to the survivor before the member is +/// removed. A Down would skip that hand-off and leave the pair to the downing strategy +/// — the wrong tool for a deliberate, planned role swap. +/// +/// The target is the oldest Up member, not the leader. That mirrors +/// ActiveNodeEvaluator's rule, which is where the singletons actually live; Akka's +/// cluster leadership is address-ordered and diverges from singleton placement after a +/// restart (review 01 [High]). +/// +/// Audit before acting. The row is written before the Leave is issued. The node +/// serving this call is usually NOT the one leaving, but it can be (an admin routed to the +/// active node fails that node over), and an audit written afterwards could be lost to the +/// very shutdown it describes. +/// +public sealed class AkkaManualFailoverService : IManualFailoverService +{ + private readonly AkkaHostedService _akka; + private readonly ICentralAuditWriter _audit; + private readonly ILogger _logger; + + /// Initializes a new . + /// The Akka hosted service exposing the cluster's actor system. + /// Central direct-write audit writer. + /// Logger. + public AkkaManualFailoverService( + AkkaHostedService akka, + ICentralAuditWriter audit, + ILogger logger) + { + _akka = akka; + _audit = audit; + _logger = logger; + } + + /// + public async Task FailOverCentralAsync(string actor) + { + var system = _akka.GetOrCreateActorSystem(); + + // Resolve first so the audit row can name the target, and so the peer guard rejects + // before anything observable happens. + var target = FailOverCore(system, role: CentralRole, dryRun: true); + if (target is null) + { + _logger.LogWarning( + "Manual failover requested by {Actor} but refused: fewer than 2 Up '{Role}' members, " + + "so there is no standby to take over.", actor, CentralRole); + return null; + } + + await WriteAuditAsync(actor, target); + + _logger.LogWarning( + "Manual failover triggered by {Actor}: {Target} is leaving the cluster gracefully; " + + "its singletons hand over to the standby, it restarts via its supervisor and rejoins " + + "as the youngest member.", actor, target); + + FailOverCore(system, role: CentralRole); + return target.ToString(); + } + + /// The Akka role scoping central-cluster membership. + private const string CentralRole = "Central"; + + /// + /// Oldest Up member with the role leaves — mirrors ActiveNodeEvaluator's oldest-Up + /// rule so the node acted on is exactly the one hosting the singletons. Returns null when + /// fewer than 2 Up members carry the role (no peer = failover would be an outage). + /// + /// The actor system whose cluster is acted on. + /// Role scope for membership. + /// When true, resolve and return the target without issuing the Leave. + /// The address that leaves (or would leave), or null when there is no peer. + public static Address? FailOverCore(ActorSystem system, string role, bool dryRun = false) + { + var cluster = Akka.Cluster.Cluster.Get(system); + var withRole = cluster.State.Members + .Where(m => m.Status == MemberStatus.Up && m.HasRole(role)) + .OrderBy(m => m, Member.AgeOrdering) + .ToList(); + + if (withRole.Count < 2) + { + return null; + } + + var oldest = withRole[0]; + if (!dryRun) + { + cluster.Leave(oldest.Address); + } + + return oldest.Address; + } + + /// + /// Best-effort audit row. Audit failure must never block the failover the admin asked for + /// — the same rule the rest of the Audit Log follows (audit is best-effort; the + /// user-facing action's own success path is authoritative). + /// + private async Task WriteAuditAsync(string actor, Address target) + { + try + { + var evt = ScadaBridgeAuditEventFactory.Create( + channel: AuditChannel.Cluster, + kind: AuditKind.ManualFailover, + status: AuditStatus.Submitted, + actor: actor, + target: target.ToString(), + extra: JsonSerializer.Serialize(new { target = target.ToString(), role = CentralRole })); + + await _audit.WriteAsync(evt); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Best-effort manual-failover audit emission failed (actor={Actor}, target={Target}); " + + "the failover itself proceeds.", actor, target); + } + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs index 5df284bc..c04bebb1 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs @@ -329,6 +329,12 @@ try // which node is active. builder.Services.AddSingleton(); + // Admin-triggered manual failover of the central pair (Health page control, + // decision 2026-07-22). Central-only: the seam is declared in CentralUI so that + // project stays Akka-free, and only this branch has a cluster to act on. + builder.Services.AddSingleton< + ZB.MOM.WW.ScadaBridge.CentralUI.Services.IManualFailoverService, AkkaManualFailoverService>(); + // Cluster node status provider scoped to the Central role — feeds the // CentralHealthReportLoop so the central cluster appears on /monitoring/health. builder.Services.AddSingleton(sp => diff --git a/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Types/Enums/AuditEnumTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Types/Enums/AuditEnumTests.cs index 00cb263a..b2176564 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Types/Enums/AuditEnumTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Types/Enums/AuditEnumTests.cs @@ -12,7 +12,7 @@ public class AuditEnumTests [Fact] public void AuditChannel_HasExactlyExpectedMembers() { - var expected = new[] { "ApiOutbound", "DbOutbound", "Notification", "ApiInbound", "SecuredWrite" }; + var expected = new[] { "ApiOutbound", "DbOutbound", "Notification", "ApiInbound", "SecuredWrite", "Cluster" }; var actual = Enum.GetValues(typeof(AuditChannel)) .Cast() .Select(x => x.ToString()) @@ -23,7 +23,7 @@ public class AuditEnumTests } [Fact] - public void AuditKind_HasExactlySixteenExpectedMembers() + public void AuditKind_HasExactlySeventeenExpectedMembers() { var expected = new[] { @@ -33,13 +33,14 @@ public class AuditEnumTests "SecuredWriteSubmit", "SecuredWriteApprove", "SecuredWriteReject", "SecuredWriteExecute", "SecuredWriteExpire", "ReconciliationAbandoned", + "ManualFailover", }; var actual = Enum.GetValues(typeof(AuditKind)) .Cast() .Select(x => x.ToString()) .ToArray(); - Assert.Equal(16, actual.Length); + Assert.Equal(17, actual.Length); Assert.Equal(expected, actual); } diff --git a/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/ManualFailoverTests.cs b/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/ManualFailoverTests.cs new file mode 100644 index 00000000..5c6983e1 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/ManualFailoverTests.cs @@ -0,0 +1,111 @@ +using Akka.Actor; +using Akka.Cluster; +using Akka.Configuration; +using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure; +using ZB.MOM.WW.ScadaBridge.Communication.ClusterState; +using ZB.MOM.WW.ScadaBridge.Host; +using ZB.MOM.WW.ScadaBridge.Host.Actors; +using ZB.MOM.WW.ScadaBridge.Host.Health; + +namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster; + +/// +/// Cluster-level proof of the admin "Trigger failover" control: the current active node +/// (the OLDEST Up member — 's rule, which is where +/// ClusterSingletonManager places singletons) leaves GRACEFULLY, so its singletons hand over +/// rather than being killed, and the survivor becomes the active node. +/// +/// Graceful Leave, never Down: a Down would skip singleton hand-off and +/// leave the pair to the downing strategy. The peer guard matters just as much — failing over +/// a single-node cluster is not a failover, it is an outage, so the service refuses. +/// +public sealed class ManualFailoverTests : IAsyncDisposable +{ + private readonly List _systems = new(); + + /// Starts a one-node cluster (self-first seed list, so it forms alone). + private ActorSystem StartSoloNode() + { + var port = TwoNodeClusterFixture.GetFreeTcpPort(); + var nodeOptions = new NodeOptions { Role = "Central", NodeHostname = "127.0.0.1", RemotingPort = port }; + var clusterOptions = new ClusterOptions + { + SeedNodes = new List { $"akka.tcp://scadabridge@127.0.0.1:{port}" }, + AllowSingleNodeCluster = true, + StableAfter = TimeSpan.FromSeconds(3), + HeartbeatInterval = TimeSpan.FromMilliseconds(500), + FailureDetectionThreshold = TimeSpan.FromSeconds(2), + MinNrOfMembers = 1, + }; + var hocon = AkkaHostedService.BuildHocon( + nodeOptions, clusterOptions, new[] { "Central" }, + TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3)); + var system = ActorSystem.Create("scadabridge", ConfigurationFactory.ParseString(hocon)); + _systems.Add(system); + return system; + } + + [Fact] + public async Task Failover_makes_the_oldest_leave_and_the_survivor_take_over() + { + await using var f = await TwoNodeClusterFixture.StartAsync(); + var oldest = Akka.Cluster.Cluster.Get(f.NodeA); // NodeA started first = oldest + Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(oldest, "Central"), + "precondition: NodeA must be the active (oldest Up) node before the failover"); + + // Issued from the OTHER node, exactly as the UI does it (the button is served by + // whichever node Traefik routed the admin to). + var target = AkkaManualFailoverService.FailOverCore(f.NodeB, "Central"); + + Assert.Equal(oldest.SelfAddress, target); + + // Graceful exit path: the left node's own ActorSystem terminates… + await f.NodeA.WhenTerminated.WaitAsync(TimeSpan.FromSeconds(30)); + // …and the survivor becomes a 1-member cluster and the oldest-Up active node. + await TwoNodeClusterFixture.WaitForMemberRemoved(f.NodeB, oldest.SelfAddress, TimeSpan.FromSeconds(30)); + Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(Akka.Cluster.Cluster.Get(f.NodeB), "Central")); + } + + [Fact] + public async Task Failover_refuses_when_no_peer_exists() + { + var solo = StartSoloNode(); + await TwoNodeClusterFixture.WaitForMembersUp(solo, 1, TimeSpan.FromSeconds(30)); + + var target = AkkaManualFailoverService.FailOverCore(solo, "Central"); + + Assert.Null(target); + // Positive assert: the refusal left the node running and still active — the guard + // must not have half-issued a Leave. + await Task.Delay(TimeSpan.FromSeconds(2)); + Assert.Equal(MemberStatus.Up, Akka.Cluster.Cluster.Get(solo).SelfMember.Status); + Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(Akka.Cluster.Cluster.Get(solo), "Central")); + } + + [Fact] + public async Task Failover_dry_run_names_the_target_without_moving_the_cluster() + { + // The service resolves the target first (to audit it BEFORE acting); that probe must + // not itself perturb the cluster. + await using var f = await TwoNodeClusterFixture.StartAsync(); + var oldest = Akka.Cluster.Cluster.Get(f.NodeA); + + var probed = AkkaManualFailoverService.FailOverCore(f.NodeB, "Central", dryRun: true); + + Assert.Equal(oldest.SelfAddress, probed); + await Task.Delay(TimeSpan.FromSeconds(3)); + Assert.Equal(2, Akka.Cluster.Cluster.Get(f.NodeB).State.Members.Count(m => m.Status == MemberStatus.Up)); + Assert.True(ActiveNodeEvaluator.SelfIsOldestUp(oldest, "Central")); + } + + public async ValueTask DisposeAsync() + { + foreach (var s in _systems) + { + if (s is { WhenTerminated.IsCompleted: false }) + { + try { await s.Terminate().WaitAsync(TimeSpan.FromSeconds(10)); } catch { /* teardown */ } + } + } + } +}