Merge R2-04 Failure-visibility trio (arch-review round 2) [PR #433]

Findings 01/S-1 (AddressSpaceApplyOutcome failure field + apply.failed logging,
no optimistic success), 06/S-1 (Galaxy write fails closed -> BadCommunicationError
+ #5 revert, no knowingly-lost raw Write), 03/S4 (PrimaryGatePolicy default-deny
unknown-role-multi-driver on all gates + scripted-alarm emit gate). T13/T15 (2-node
live gates) deferred -- T15 is the behavior-affecting S4 live gate, must run in heavy
pass. Clean merge, build clean.

src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs

@@ -58,10 +58,24 @@ public static class OtOpcUaTelemetry
         Meter.CreateCounter<long>("otopcua.opcua.sink.write", unit: "{write}",
             description: "Writes that landed in IOpcUaAddressSpaceSink (kind=value|alarm|rebuild).");
 
+    /// <summary>Address-space apply/materialise passes that swallowed at least one sink failure —
+    /// a failed rebuild (kind=rebuild) or per-node materialise failures (kind=nodes). A non-zero rate
+    /// means the running server holds a stale or partially-materialised address space despite a
+    /// reported-successful deploy (archreview 01/S-1).</summary>
+    public static readonly Counter<long> OpcUaApplyFailed =
+        Meter.CreateCounter<long>("otopcua.opcua.apply.failed", unit: "{apply}",
+            description: "Apply/materialise passes with swallowed sink failures (kind=rebuild|nodes).");
+
     public static readonly Counter<long> ServiceLevelChange =
         Meter.CreateCounter<long>("otopcua.redundancy.service_level_change", unit: "{change}",
             description: "OPC UA Server.ServiceLevel transitions emitted by the redundancy state.");
 
+    /// <summary>Inbound operations denied by the Primary data-plane gate (archreview 03/S4) —
+    /// site=write|ack|alarm-emit, reason=secondary|detached|role-unknown.</summary>
+    public static readonly Counter<long> PrimaryGateDenied =
+        Meter.CreateCounter<long>("otopcua.redundancy.primary_gate_denied", unit: "{denial}",
+            description: "Operations denied by the Primary gate (site=write|ack|alarm-emit, reason=secondary|detached|role-unknown).");
+
     // ---------------- Convenience helpers ----------------
 
     /// <summary>

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/GatewayGalaxyDataWriter.cs

@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using System.Diagnostics.Metrics;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using ZB.MOM.WW.MxGateway.Client;
@@ -37,6 +38,18 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
     private readonly ConcurrentDictionary<int, byte> _supervisedHandles = new();
     private int _addItemCallCount;
 
+    // Write failure-visibility meters (archreview 06/S-1). Reuse the pump's meter name so the single host
+    // listener catches both. advise_failed = writes short-circuited to Bad because AdviseSupervisory failed
+    // (the value could not have committed); unconfirmed = writes reported Good off an EMPTY gateway statuses
+    // array (command accepted, COM commit unconfirmed pending the gateway WriteComplete correlation follow-up).
+    private static readonly Meter WriterMeter = new(EventPump.MeterName);
+    private static readonly Counter<long> WriteAdviseFailed =
+        WriterMeter.CreateCounter<long>("galaxy.writes.advise_failed", unit: "{write}",
+            description: "Writes short-circuited to Bad because AdviseSupervisory failed (value could not have committed).");
+    private static readonly Counter<long> WriteUnconfirmed =
+        WriterMeter.CreateCounter<long>("galaxy.writes.unconfirmed", unit: "{write}",
+            description: "Writes reported Good off an EMPTY gateway statuses array (command accepted; commit unconfirmed).");
+
     /// <summary>Initializes a new Galaxy data writer.</summary>
     /// <param name="session">The MXAccess gateway session.</param>
     /// <param name="writeUserId">The user ID for write operations.</param>
@@ -129,23 +142,39 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
                 "GalaxyMxSession is not connected. Call ConnectAsync before issuing writes.");
         var serverHandle = _session.ServerHandle;
 
+        // Bind the per-batch MX write operations behind the internal IMxWriteOps seam (see below) so the
+        // whole write pipeline is unit-testable without the sealed SDK session (archreview 06/S-1).
+        var ops = new SessionMxWriteOps(session, serverHandle);
+
         var results = new WriteResult[writes.Count];
         for (var i = 0; i < writes.Count; i++)
         {
-            results[i] = await WriteOneAsync(session, serverHandle, writes[i],
+            results[i] = await WriteOneAsync(ops, writes[i],
                 securityResolver(writes[i].FullReference), cancellationToken)
                 .ConfigureAwait(false);
         }
         return results;
     }
 
+    /// <summary>Test seam: drive the real single-write pipeline against a fake <see cref="IMxWriteOps"/>
+    /// so the fail-closed / unconfirmed behaviour (archreview 06/S-1) is covered offline. Behaviourally
+    /// identical to the per-entry path <see cref="WriteAsync"/> runs.</summary>
+    /// <param name="ops">The (fake) MX write operations.</param>
+    /// <param name="request">The write request.</param>
+    /// <param name="classification">The tag's security classification.</param>
+    /// <param name="ct">The cancellation token.</param>
+    /// <returns>The translated write result.</returns>
+    internal Task<WriteResult> WriteOneForTestAsync(
+        IMxWriteOps ops, WriteRequest request, SecurityClassification classification, CancellationToken ct)
+        => WriteOneAsync(ops, request, classification, ct);
+
     private async Task<WriteResult> WriteOneAsync(
-        MxGatewaySession session, int serverHandle, WriteRequest request,
+        IMxWriteOps ops, WriteRequest request,
         SecurityClassification classification, CancellationToken ct)
     {
         try
         {
-            var itemHandle = await EnsureItemHandleAsync(session, serverHandle, request.FullReference, ct)
+            var itemHandle = await EnsureItemHandleAsync(ops, request.FullReference, ct)
                 .ConfigureAwait(false);
             var mxValue = MxValueEncoder.Encode(request.Value);
 
@@ -154,7 +183,7 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
             {
                 // SecuredWrite/VerifiedWrite tags carry their own ArchestrA user identity
                 // (current/verifier user), so they don't use the supervisory path.
-                reply = await InvokeWriteSecuredAsync(session, serverHandle, itemHandle, mxValue, ct)
+                reply = await InvokeWriteSecuredAsync(ops, itemHandle, mxValue, ct)
                     .ConfigureAwait(false);
             }
             else
@@ -162,9 +191,18 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
                 // A raw Write runs with NO user login (WriteUserId is typically 0), so MXAccess
                 // only COMMITS the value when the item is advised in SUPERVISORY mode. Without it
                 // the gateway's Write call doesn't throw (reply looks OK) but the value never
-                // reaches the galaxy. AdviseSupervisory once per handle, then Write.
-                await EnsureSupervisoryAdvisedAsync(session, serverHandle, itemHandle, ct).ConfigureAwait(false);
-                reply = await session.WriteRawAsync(serverHandle, itemHandle, mxValue, _writeUserId, ct)
+                // reaches the galaxy — a knowingly-lost write. FAIL-CLOSED (archreview 06/S-1): when
+                // AdviseSupervisory fails we do NOT issue the raw Write; we return Bad so the
+                // #5 write-outcome self-correction reverts the phantom-Good node instead of leaving it.
+                // The real fix is a gateway-side WriteComplete correlation (mxaccessgw backlog) so the
+                // unary reply's Statuses carry the actual COM commit outcome; until then advise-status +
+                // the galaxy.writes.* meters are the honest signal.
+                if (!await EnsureSupervisoryAdvisedAsync(ops, itemHandle, ct).ConfigureAwait(false))
+                {
+                    WriteAdviseFailed.Add(1);
+                    return new WriteResult(StatusCodeMap.BadCommunicationError);
+                }
+                reply = await ops.WriteRawAsync(itemHandle, mxValue, _writeUserId, ct)
                     .ConfigureAwait(false);
             }
 
@@ -189,10 +227,10 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
         classification is SecurityClassification.SecuredWrite or SecurityClassification.VerifiedWrite;
 
     private async Task<int> EnsureItemHandleAsync(
-        MxGatewaySession session, int serverHandle, string fullRef, CancellationToken ct)
+        IMxWriteOps ops, string fullRef, CancellationToken ct)
     {
         if (TryResolveCachedOrBorrowed(fullRef) is int resolved) return resolved;
-        var handle = await session.AddItemAsync(serverHandle, fullRef, ct).ConfigureAwait(false);
+        var handle = await ops.AddItemAsync(fullRef, ct).ConfigureAwait(false);
         Interlocked.Increment(ref _addItemCallCount);
         _itemHandles[fullRef] = handle;
         return handle;
@@ -210,36 +248,45 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
     ///     session reconnect via <see cref="InvalidateHandleCaches"/>, so a fresh session always
     ///     re-advises — the supervisory state never outlives the handle it was taken against.
     /// </summary>
-    private async Task EnsureSupervisoryAdvisedAsync(
-        MxGatewaySession session, int serverHandle, int itemHandle, CancellationToken ct)
+    /// <summary>Ensure the item is supervisory-advised so a no-login raw Write can commit. Returns
+    /// <c>true</c> when the item is advised (already-advised this session, or the advise round-trip
+    /// succeeded), <c>false</c> when the advise failed — the caller then FAILS THE WRITE CLOSED
+    /// (archreview 06/S-1) rather than issuing a raw Write that cannot commit.</summary>
+    /// <returns><c>true</c> when advised; <c>false</c> when the advise failed (caller must not write).</returns>
+    private async Task<bool> EnsureSupervisoryAdvisedAsync(
+        IMxWriteOps ops, int itemHandle, CancellationToken ct)
     {
-        if (!_supervisedHandles.TryAdd(itemHandle, 0)) return;
+        // Already advised this session (a prior successful advise on this handle) — nothing to do.
+        if (!_supervisedHandles.TryAdd(itemHandle, 0)) return true;
 
         var request = new MxCommandRequest
         {
-            SessionId = session.SessionId,
+            SessionId = ops.SessionId,
             ClientCorrelationId = Guid.NewGuid().ToString("N"),
             Command = new MxCommand
             {
                 Kind = MxCommandKind.AdviseSupervisory,
                 AdviseSupervisory = new AdviseSupervisoryCommand
                 {
-                    ServerHandle = serverHandle,
+                    ServerHandle = ops.ServerHandle,
                     ItemHandle = itemHandle,
                 },
             },
         };
 
-        var reply = await session.InvokeAsync(request, ct).ConfigureAwait(false);
+        var reply = await ops.InvokeAsync(request, ct).ConfigureAwait(false);
         if (reply.ProtocolStatus is { } proto && proto.Code != ProtocolStatusCode.Ok)
         {
-            // Supervisory advise failed — forget it so the next write retries, and let the
-            // write proceed (it surfaces its own status via TranslateReply).
+            // Supervisory advise failed — forget it so the next write retries the advise, and report
+            // failure so the caller fails the write closed (the raw Write would not commit).
             _supervisedHandles.TryRemove(itemHandle, out _);
             _logger.LogWarning(
                 "GalaxyDriver supervisory advise failed for item {ItemHandle}: {Code} {Message}",
                 itemHandle, proto.Code, proto.Message);
+            return false;
         }
+
+        return true;
     }
 
     /// <summary>
@@ -250,14 +297,14 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
     ///     interprets the underlying MXAccess command kind.
     /// </summary>
     private static Task<MxCommandReply> InvokeWriteSecuredAsync(
-        MxGatewaySession session, int serverHandle, int itemHandle, MxValue value, CancellationToken ct)
+        IMxWriteOps ops, int itemHandle, MxValue value, CancellationToken ct)
     {
         var command = new MxCommand
         {
             Kind = MxCommandKind.WriteSecured,
             WriteSecured = new WriteSecuredCommand
             {
-                ServerHandle = serverHandle,
+                ServerHandle = ops.ServerHandle,
                 ItemHandle = itemHandle,
                 Value = value,
                 CurrentUserId = 0,
@@ -266,17 +313,22 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
         };
         var request = new MxCommandRequest
         {
-            SessionId = session.SessionId,
+            SessionId = ops.SessionId,
             ClientCorrelationId = Guid.NewGuid().ToString("N"),
             Command = command,
         };
-        return session.InvokeAsync(request, ct);
+        return ops.InvokeAsync(request, ct);
     }
 
     /// <summary>
     ///     Translate a gateway <see cref="MxCommandReply"/> into an OPC UA
     ///     <see cref="WriteResult"/>. Honours the protocol-level Status field first
     ///     (transport / dispatch failures), then the first MXAccess status row.
+    ///     <para>An EMPTY statuses array is treated as <c>Good</c> but is PROVISIONAL: the reply proves
+    ///     worker-side command acceptance, not the COM-side commit (the gateway does not yet correlate a
+    ///     WriteComplete row onto the unary reply — mxaccessgw backlog). Each such reply increments
+    ///     <c>galaxy.writes.unconfirmed</c> so the unconfirmed-write rate is operator-visible today
+    ///     (archreview 06/S-1).</para>
     /// </summary>
     private WriteResult TranslateReply(MxCommandReply reply, string fullRef)
     {
@@ -298,6 +350,81 @@ public sealed class GatewayGalaxyDataWriter : IGalaxyDataWriter
             return new WriteResult(StatusCodeMap.FromMxStatus(status, _logger));
         }
 
+        // Empty statuses ⇒ provisional Good. Meter it so the unconfirmed-write rate is visible until the
+        // gateway-side WriteComplete correlation lands (archreview 06/S-1 cross-repo follow-up).
+        WriteUnconfirmed.Add(1);
         return new WriteResult(StatusCodeMap.Good);
     }
 }
+
+/// <summary>
+///     The per-write MXAccess gateway operations the writer needs, extracted behind an internal seam so the
+///     fail-closed write pipeline (archreview 06/S-1) is unit-testable — the SDK <c>MxGatewaySession</c>
+///     types are sealed with internal ctors and cannot be faked. Production binds
+///     <see cref="SessionMxWriteOps"/> over the real <c>(MxGatewaySession, serverHandle)</c>; tests inject a
+///     fake that records calls and returns canned replies.
+/// </summary>
+internal interface IMxWriteOps
+{
+    /// <summary>The gateway session id stamped onto every <see cref="MxCommandRequest"/>.</summary>
+    string SessionId { get; }
+
+    /// <summary>The MXAccess server handle carried in each item/advise/write command.</summary>
+    int ServerHandle { get; }
+
+    /// <summary>Add an MXAccess item and return its item handle.</summary>
+    /// <param name="fullRef">The dotted tag full reference.</param>
+    /// <param name="ct">The cancellation token.</param>
+    /// <returns>The MXAccess item handle.</returns>
+    Task<int> AddItemAsync(string fullRef, CancellationToken ct);
+
+    /// <summary>Invoke a raw <see cref="MxCommand"/> (AdviseSupervisory / WriteSecured) and return the reply.</summary>
+    /// <param name="request">The command request.</param>
+    /// <param name="ct">The cancellation token.</param>
+    /// <returns>The gateway reply.</returns>
+    Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken ct);
+
+    /// <summary>Issue a raw (non-secured) Write for the given item handle.</summary>
+    /// <param name="itemHandle">The MXAccess item handle to write.</param>
+    /// <param name="value">The encoded MX value.</param>
+    /// <param name="userId">The write user id (typically 0 — the supervisory path commits it).</param>
+    /// <param name="ct">The cancellation token.</param>
+    /// <returns>The gateway reply.</returns>
+    Task<MxCommandReply> WriteRawAsync(int itemHandle, MxValue value, int userId, CancellationToken ct);
+}
+
+/// <summary>Production <see cref="IMxWriteOps"/> — a thin adapter over a live
+/// <see cref="MxGatewaySession"/> and its server handle. Behaviour is bit-identical to the previous
+/// inline <c>(session, serverHandle)</c> calls.</summary>
+internal sealed class SessionMxWriteOps : IMxWriteOps
+{
+    private readonly MxGatewaySession _session;
+    private readonly int _serverHandle;
+
+    /// <summary>Initializes the adapter over a connected session + server handle.</summary>
+    /// <param name="session">The connected gateway session.</param>
+    /// <param name="serverHandle">The MXAccess server handle for this session.</param>
+    public SessionMxWriteOps(MxGatewaySession session, int serverHandle)
+    {
+        _session = session;
+        _serverHandle = serverHandle;
+    }
+
+    /// <inheritdoc />
+    public string SessionId => _session.SessionId;
+
+    /// <inheritdoc />
+    public int ServerHandle => _serverHandle;
+
+    /// <inheritdoc />
+    public Task<int> AddItemAsync(string fullRef, CancellationToken ct)
+        => _session.AddItemAsync(_serverHandle, fullRef, ct);
+
+    /// <inheritdoc />
+    public Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken ct)
+        => _session.InvokeAsync(request, ct);
+
+    /// <inheritdoc />
+    public Task<MxCommandReply> WriteRawAsync(int itemHandle, MxValue value, int userId, CancellationToken ct)
+        => _session.WriteRawAsync(_serverHandle, itemHandle, value, userId, ct);
+}

src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/AddressSpaceApplier.cs

@@ -82,14 +82,17 @@ public sealed class AddressSpaceApplier
 
         var ts = DateTime.UtcNow;
         var removedCount = 0;
+        // Swallowed removal-pass condition-write failures are tallied here (not lost to a per-node Warning)
+        // so a degraded removal is operator-visible via AddressSpaceApplyOutcome.FailedNodes (archreview 01/S-1).
+        var failedNodes = 0;
         foreach (var eq in plan.RemovedEquipment)
         {
-            SafeWriteAlarmCondition(eq.EquipmentId, RemovedConditionState, ts);
+            if (!SafeWriteAlarmCondition(eq.EquipmentId, RemovedConditionState, ts)) failedNodes++;
             removedCount++;
         }
         foreach (var alarm in plan.RemovedAlarms)
         {
-            SafeWriteAlarmCondition(alarm.ScriptedAlarmId, RemovedConditionState, ts);
+            if (!SafeWriteAlarmCondition(alarm.ScriptedAlarmId, RemovedConditionState, ts)) failedNodes++;
             removedCount++;
         }
         // Removed equipment tags / VirtualTags are plain variable nodes (no Part 9 condition to write
@@ -146,10 +149,11 @@ public sealed class AddressSpaceApplier
         // covered for free and need no separate surgical pass.
         var renamedFolders = plan.RenamedFolders;
         var rebuilt = false;
+        var rebuildFailed = false;
 
         if (structuralRebuild)
         {
-            SafeRebuild();
+            rebuildFailed = !SafeRebuild();
             rebuilt = true;
         }
         else if (surgicalTagDeltas.Count > 0 || renamedFolders.Count > 0)
@@ -188,12 +192,12 @@ public sealed class AddressSpaceApplier
                     }
                     if (!ok) { allApplied = false; break; }
                 }
-                if (!allApplied) { SafeRebuild(); rebuilt = true; }
+                if (!allApplied) { rebuildFailed = !SafeRebuild(); rebuilt = true; }
             }
             else
             {
                 // Sink lacks the surgical capability ⇒ rebuild (safe default).
-                SafeRebuild();
+                rebuildFailed = !SafeRebuild();
                 rebuilt = true;
             }
         }
@@ -212,7 +216,7 @@ public sealed class AddressSpaceApplier
         // currently-historized set. Same non-blocking + throw-safe discipline as the provisioning hook.
         FeedHistorizedRefs(plan);
 
-        return new AddressSpaceApplyOutcome(removedCount, addedCount, changedCount, rebuilt);
+        return new AddressSpaceApplyOutcome(removedCount, addedCount, changedCount, rebuilt, rebuildFailed, failedNodes);
     }
 
     /// <summary>
@@ -370,15 +374,22 @@ public sealed class AddressSpaceApplier
                 string.IsNullOrWhiteSpace(tag.HistorianTagname) ? tag.FullName : tag.HistorianTagname)
             : null;
 
-    private void SafeRebuild()
+    /// <summary>Rebuild the sink's address space, swallowing (and Error-logging) any fault.
+    /// Returns <c>true</c> on success, <c>false</c> when the sink threw — the caller threads the
+    /// result into <see cref="AddressSpaceApplyOutcome.RebuildFailed"/> so a broken rebuild is
+    /// visible instead of reported as optimistic success (archreview 01/S-1).</summary>
+    /// <returns><c>true</c> when the rebuild completed; <c>false</c> when the sink threw.</returns>
+    private bool SafeRebuild()
     {
         try
         {
             _sink.RebuildAddressSpace();
+            return true;
         }
         catch (Exception ex)
         {
             _logger.LogError(ex, "AddressSpaceApplier: sink.RebuildAddressSpace threw");
+            return false;
         }
     }
 
@@ -390,28 +401,32 @@ public sealed class AddressSpaceApplier
     ///     present, so re-applies are cheap.
     /// </summary>
     /// <param name="composition">The composition result containing the hierarchy to materialise.</param>
-    public void MaterialiseHierarchy(AddressSpaceComposition composition)
+    /// <returns>The count of swallowed sink failures in this pass (0 on a clean apply) — the publish actor
+    /// sums it into the degraded-apply surface (archreview 01/S-1).</returns>
+    public int MaterialiseHierarchy(AddressSpaceComposition composition)
     {
         ArgumentNullException.ThrowIfNull(composition);
 
+        var failed = 0;
         foreach (var area in composition.UnsAreas)
         {
-            SafeEnsureFolder(area.UnsAreaId, parentNodeId: null, displayName: area.DisplayName);
+            if (!SafeEnsureFolder(area.UnsAreaId, parentNodeId: null, displayName: area.DisplayName)) failed++;
         }
         foreach (var line in composition.UnsLines)
         {
-            SafeEnsureFolder(line.UnsLineId, parentNodeId: line.UnsAreaId, displayName: line.DisplayName);
+            if (!SafeEnsureFolder(line.UnsLineId, parentNodeId: line.UnsAreaId, displayName: line.DisplayName)) failed++;
         }
         foreach (var equipment in composition.EquipmentNodes)
         {
             // Equipment with no UnsLineId (legacy / dev rows) hang under the root.
             var parent = string.IsNullOrWhiteSpace(equipment.UnsLineId) ? null : equipment.UnsLineId;
-            SafeEnsureFolder(equipment.EquipmentId, parentNodeId: parent, displayName: equipment.DisplayName);
+            if (!SafeEnsureFolder(equipment.EquipmentId, parentNodeId: parent, displayName: equipment.DisplayName)) failed++;
         }
 
         _logger.LogInformation(
-            "AddressSpaceApplier: hierarchy materialised (areas={Areas}, lines={Lines}, equipment={Equipment})",
-            composition.UnsAreas.Count, composition.UnsLines.Count, composition.EquipmentNodes.Count);
+            "AddressSpaceApplier: hierarchy materialised (areas={Areas}, lines={Lines}, equipment={Equipment}, failed={Failed})",
+            composition.UnsAreas.Count, composition.UnsLines.Count, composition.EquipmentNodes.Count, failed);
+        return failed;
     }
 
     /// <summary>
@@ -429,11 +444,14 @@ public sealed class AddressSpaceApplier
     ///     takes a plain <c>string dataType</c> (not a DriverAttributeInfo).
     /// </summary>
     /// <param name="composition">The composition result containing the equipment tags to materialise.</param>
-    public void MaterialiseEquipmentTags(AddressSpaceComposition composition)
+    /// <returns>The count of swallowed sink failures in this pass (0 on a clean apply) — the publish actor
+    /// sums it into the degraded-apply surface (archreview 01/S-1).</returns>
+    public int MaterialiseEquipmentTags(AddressSpaceComposition composition)
     {
         ArgumentNullException.ThrowIfNull(composition);
-        if (composition.EquipmentTags.Count == 0) return;
+        if (composition.EquipmentTags.Count == 0) return 0;
 
+        var failed = 0;
         // Sub-folders first — a tag's FolderPath becomes one folder UNDER its equipment folder
         // (deduped per distinct equipment+path). Tags with no FolderPath hang directly under the
         // equipment folder, which MaterialiseHierarchy already created (never re-create
@@ -444,7 +462,7 @@ public sealed class AddressSpaceApplier
             if (string.IsNullOrWhiteSpace(tag.FolderPath)) continue;
             var folderNodeId = EquipmentNodeIds.SubFolder(tag.EquipmentId, tag.FolderPath);
             if (!foldersCreated.Add(folderNodeId)) continue;
-            SafeEnsureFolder(folderNodeId, parentNodeId: tag.EquipmentId, displayName: tag.FolderPath);
+            if (!SafeEnsureFolder(folderNodeId, parentNodeId: tag.EquipmentId, displayName: tag.FolderPath)) failed++;
         }
 
         // Variables: NodeId is FOLDER-SCOPED ("<parent>/<Name>"), NOT the raw FullName — a driver
@@ -463,7 +481,7 @@ public sealed class AddressSpaceApplier
             {
                 // Native alarm tag → a real Part 9 condition node (reuses the scripted-alarm path),
                 // NOT a value variable. Parent is the sub-folder when set, else the equipment folder.
-                SafeMaterialiseAlarmCondition(nodeId, parent, tag.Name, tag.Alarm.AlarmType, tag.Alarm.Severity, isNative: true);
+                if (!SafeMaterialiseAlarmCondition(nodeId, parent, tag.Name, tag.Alarm.AlarmType, tag.Alarm.Severity, isNative: true)) failed++;
             }
             else
             {
@@ -477,14 +495,16 @@ public sealed class AddressSpaceApplier
                 // even if authored ReadWrite, so a client write cannot reach the driver write path which
                 // does not handle arrays (e.g. S7 BoxValueForWrite would crash).
                 var writable = tag.Writable && !tag.IsArray;
-                SafeEnsureVariable(nodeId, parent, tag.Name, tag.DataType, writable, historianTagname, tag.IsArray, tag.ArrayLength);
+                if (!SafeEnsureVariable(nodeId, parent, tag.Name, tag.DataType, writable, historianTagname, tag.IsArray, tag.ArrayLength)) failed++;
             }
         }
 
         _logger.LogInformation(
-            "AddressSpaceApplier: equipment tags materialised (tags={Tags}, equipment={Equipment})",
+            "AddressSpaceApplier: equipment tags materialised (tags={Tags}, equipment={Equipment}, failed={Failed})",
             composition.EquipmentTags.Count,
-            composition.EquipmentTags.Select(t => t.EquipmentId).Distinct(StringComparer.Ordinal).Count());
+            composition.EquipmentTags.Select(t => t.EquipmentId).Distinct(StringComparer.Ordinal).Count(),
+            failed);
+        return failed;
     }
 
     /// <summary>
@@ -498,7 +518,9 @@ public sealed class AddressSpaceApplier
     /// NodeAdded model-change is announced under this node.</param>
     /// <param name="folders">The discovered folders to ensure (parent-first by depth).</param>
     /// <param name="variables">The discovered variables to ensure (read-only value nodes).</param>
-    public void MaterialiseDiscoveredNodes(
+    /// <returns>The count of swallowed sink failures in this pass (0 on a clean apply) — the publish actor
+    /// surfaces a non-zero count as a degraded discovered-node injection (archreview 01/S-1).</returns>
+    public int MaterialiseDiscoveredNodes(
         string equipmentRootNodeId,
         IReadOnlyList<DiscoveredFolder> folders,
         IReadOnlyList<DiscoveredVariable> variables)
@@ -506,25 +528,27 @@ public sealed class AddressSpaceApplier
         ArgumentException.ThrowIfNullOrEmpty(equipmentRootNodeId);
         ArgumentNullException.ThrowIfNull(folders);
         ArgumentNullException.ThrowIfNull(variables);
-        if (folders.Count == 0 && variables.Count == 0) return;
+        if (folders.Count == 0 && variables.Count == 0) return 0;
 
+        var failed = 0;
         // Parent-first: a child folder's parent must exist before it. Ordering by '/' count == depth.
         foreach (var f in folders.OrderBy(f => f.NodeId.Count(c => c == '/')))
-            SafeEnsureFolder(f.NodeId, f.ParentNodeId, f.DisplayName);
+            if (!SafeEnsureFolder(f.NodeId, f.ParentNodeId, f.DisplayName)) failed++;
 
         foreach (var v in variables)
         {
             // Mirror MaterialiseEquipmentTags: arrays forced read-only (the driver write path can't handle arrays).
             var writable = v.Writable && !v.IsArray;
-            SafeEnsureVariable(v.NodeId, v.ParentNodeId, v.DisplayName, v.DataType, writable,
-                historianTagname: null, isArray: v.IsArray, arrayLength: v.ArrayLength);
+            if (!SafeEnsureVariable(v.NodeId, v.ParentNodeId, v.DisplayName, v.DataType, writable,
+                historianTagname: null, isArray: v.IsArray, arrayLength: v.ArrayLength)) failed++;
         }
 
         _sink.RaiseNodesAddedModelChange(equipmentRootNodeId);
 
         _logger.LogInformation(
-            "AddressSpaceApplier: discovered nodes materialised under {Equipment} (folders={Folders}, vars={Vars})",
-            equipmentRootNodeId, folders.Count, variables.Count);
+            "AddressSpaceApplier: discovered nodes materialised under {Equipment} (folders={Folders}, vars={Vars}, failed={Failed})",
+            equipmentRootNodeId, folders.Count, variables.Count, failed);
+        return failed;
     }
 
     /// <summary>
@@ -540,11 +564,14 @@ public sealed class AddressSpaceApplier
     ///     Idempotent (per-variable idempotency relies on the sink's own <c>EnsureVariable</c>).
     /// </summary>
     /// <param name="composition">The composition result containing the equipment VirtualTags to materialise.</param>
-    public void MaterialiseEquipmentVirtualTags(AddressSpaceComposition composition)
+    /// <returns>The count of swallowed sink failures in this pass (0 on a clean apply) — the publish actor
+    /// sums it into the degraded-apply surface (archreview 01/S-1).</returns>
+    public int MaterialiseEquipmentVirtualTags(AddressSpaceComposition composition)
     {
         ArgumentNullException.ThrowIfNull(composition);
-        if (composition.EquipmentVirtualTags.Count == 0) return;
+        if (composition.EquipmentVirtualTags.Count == 0) return 0;
 
+        var failed = 0;
         // Sub-folders first — a VirtualTag's FolderPath becomes one folder UNDER its equipment folder
         // (deduped per distinct equipment+path). VirtualTags with no FolderPath hang directly under the
         // equipment folder, which MaterialiseHierarchy already created (never re-create it here).
@@ -554,7 +581,7 @@ public sealed class AddressSpaceApplier
             if (string.IsNullOrWhiteSpace(v.FolderPath)) continue;
             var folderNodeId = EquipmentNodeIds.SubFolder(v.EquipmentId, v.FolderPath);
             if (!foldersCreated.Add(folderNodeId)) continue;
-            SafeEnsureFolder(folderNodeId, parentNodeId: v.EquipmentId, displayName: v.FolderPath);
+            if (!SafeEnsureFolder(folderNodeId, parentNodeId: v.EquipmentId, displayName: v.FolderPath)) failed++;
         }
 
         // Variables: NodeId is FOLDER-SCOPED ("<parent>/<Name>"), mirroring the equipment-tag pass.
@@ -570,13 +597,15 @@ public sealed class AddressSpaceApplier
                 : EquipmentNodeIds.SubFolder(v.EquipmentId, v.FolderPath);
             var nodeId = EquipmentNodeIds.Variable(v.EquipmentId, v.FolderPath, v.Name);
             // VirtualTags are computed outputs — read-only nodes (no inbound write).
-            SafeEnsureVariable(nodeId, parent, v.Name, v.DataType, writable: false);
+            if (!SafeEnsureVariable(nodeId, parent, v.Name, v.DataType, writable: false)) failed++;
         }
 
         _logger.LogInformation(
-            "AddressSpaceApplier: equipment virtualtags materialised (vtags={Vtags}, equipment={Equipment})",
+            "AddressSpaceApplier: equipment virtualtags materialised (vtags={Vtags}, equipment={Equipment}, failed={Failed})",
             composition.EquipmentVirtualTags.Count,
-            composition.EquipmentVirtualTags.Select(v => v.EquipmentId).Distinct(StringComparer.Ordinal).Count());
+            composition.EquipmentVirtualTags.Select(v => v.EquipmentId).Distinct(StringComparer.Ordinal).Count(),
+            failed);
+        return failed;
     }
 
     /// <summary>
@@ -589,36 +618,49 @@ public sealed class AddressSpaceApplier
     ///     <c>MaterialiseAlarmCondition</c> re-creates cleanly on re-apply).
     /// </summary>
     /// <param name="composition">The composition result containing the scripted alarms to materialise.</param>
-    public void MaterialiseScriptedAlarms(AddressSpaceComposition composition)
+    /// <returns>The count of swallowed sink failures in this pass (0 on a clean apply) — the publish actor
+    /// sums it into the degraded-apply surface (archreview 01/S-1).</returns>
+    public int MaterialiseScriptedAlarms(AddressSpaceComposition composition)
     {
         ArgumentNullException.ThrowIfNull(composition);
-        if (composition.EquipmentScriptedAlarms.Count == 0) return;
+        if (composition.EquipmentScriptedAlarms.Count == 0) return 0;
 
         var materialised = 0;
+        var failed = 0;
         foreach (var alarm in composition.EquipmentScriptedAlarms)
         {
             if (!alarm.Enabled) continue;
-            SafeMaterialiseAlarmCondition(alarm.ScriptedAlarmId, alarm.EquipmentId, alarm.Name, alarm.AlarmType, alarm.Severity, isNative: false);
+            if (!SafeMaterialiseAlarmCondition(alarm.ScriptedAlarmId, alarm.EquipmentId, alarm.Name, alarm.AlarmType, alarm.Severity, isNative: false)) failed++;
             materialised++;
         }
 
         _logger.LogInformation(
-            "AddressSpaceApplier: scripted alarms materialised (alarms={Alarms}, equipment={Equipment})",
+            "AddressSpaceApplier: scripted alarms materialised (alarms={Alarms}, equipment={Equipment}, failed={Failed})",
             materialised,
             composition.EquipmentScriptedAlarms.Where(a => a.Enabled)
-                .Select(a => a.EquipmentId).Distinct(StringComparer.Ordinal).Count());
+                .Select(a => a.EquipmentId).Distinct(StringComparer.Ordinal).Count(),
+            failed);
+        return failed;
     }
 
-    private void SafeEnsureFolder(string nodeId, string? parentNodeId, string displayName)
+    /// <summary>Ensure a folder node, swallowing (and Warning-logging) any sink fault. Returns <c>true</c>
+    /// on success, <c>false</c> when the sink threw — callers tally the false into their pass's failed-node
+    /// count so a swallowed materialise failure is operator-visible (archreview 01/S-1).</summary>
+    /// <returns><c>true</c> when the folder was ensured; <c>false</c> when the sink threw.</returns>
+    private bool SafeEnsureFolder(string nodeId, string? parentNodeId, string displayName)
     {
-        try { _sink.EnsureFolder(nodeId, parentNodeId, displayName); }
-        catch (Exception ex) { _logger.LogWarning(ex, "AddressSpaceApplier: EnsureFolder threw for {Node}", nodeId); }
+        try { _sink.EnsureFolder(nodeId, parentNodeId, displayName); return true; }
+        catch (Exception ex) { _logger.LogWarning(ex, "AddressSpaceApplier: EnsureFolder threw for {Node}", nodeId); return false; }
     }
 
-    private void SafeEnsureVariable(string nodeId, string? parentNodeId, string displayName, string dataType, bool writable, string? historianTagname = null, bool isArray = false, uint? arrayLength = null)
+    /// <summary>Ensure a variable node, swallowing (and Warning-logging) any sink fault. Returns <c>true</c>
+    /// on success, <c>false</c> when the sink threw — callers tally the false into their pass's failed-node
+    /// count (archreview 01/S-1).</summary>
+    /// <returns><c>true</c> when the variable was ensured; <c>false</c> when the sink threw.</returns>
+    private bool SafeEnsureVariable(string nodeId, string? parentNodeId, string displayName, string dataType, bool writable, string? historianTagname = null, bool isArray = false, uint? arrayLength = null)
     {
-        try { _sink.EnsureVariable(nodeId, parentNodeId, displayName, dataType, writable, historianTagname, isArray, arrayLength); }
-        catch (Exception ex) { _logger.LogWarning(ex, "AddressSpaceApplier: EnsureVariable threw for {Node}", nodeId); }
+        try { _sink.EnsureVariable(nodeId, parentNodeId, displayName, dataType, writable, historianTagname, isArray, arrayLength); return true; }
+        catch (Exception ex) { _logger.LogWarning(ex, "AddressSpaceApplier: EnsureVariable threw for {Node}", nodeId); return false; }
     }
 
     // A VirtualTag's materialised OPC UA node (MaterialiseEquipmentVirtualTags) is derived ONLY from
@@ -674,22 +716,39 @@ public sealed class AddressSpaceApplier
         Severity: 0,
         Message: string.Empty);
 
-    private void SafeWriteAlarmCondition(string nodeId, AlarmConditionSnapshot state, DateTime ts)
+    /// <summary>Write an alarm-condition snapshot, swallowing (and Warning-logging) any sink fault.
+    /// Returns <c>true</c> on success, <c>false</c> when the sink threw — the removal pass tallies the
+    /// false into <see cref="AddressSpaceApplyOutcome.FailedNodes"/> (archreview 01/S-1).</summary>
+    /// <returns><c>true</c> when the write landed; <c>false</c> when the sink threw.</returns>
+    private bool SafeWriteAlarmCondition(string nodeId, AlarmConditionSnapshot state, DateTime ts)
     {
-        try { _sink.WriteAlarmCondition(nodeId, state, ts); }
-        catch (Exception ex) { _logger.LogWarning(ex, "AddressSpaceApplier: WriteAlarmCondition threw for {Node}", nodeId); }
+        try { _sink.WriteAlarmCondition(nodeId, state, ts); return true; }
+        catch (Exception ex) { _logger.LogWarning(ex, "AddressSpaceApplier: WriteAlarmCondition threw for {Node}", nodeId); return false; }
     }
 
-    private void SafeMaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, bool isNative)
+    /// <summary>Materialise an alarm-condition node, swallowing (and Warning-logging) any sink fault.
+    /// Returns <c>true</c> on success, <c>false</c> when the sink threw — callers tally the false into
+    /// their pass's failed-node count (archreview 01/S-1).</summary>
+    /// <returns><c>true</c> when the condition was materialised; <c>false</c> when the sink threw.</returns>
+    private bool SafeMaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, bool isNative)
     {
-        try { _sink.MaterialiseAlarmCondition(alarmNodeId, equipmentNodeId, displayName, alarmType, severity, isNative); }
-        catch (Exception ex) { _logger.LogWarning(ex, "AddressSpaceApplier: MaterialiseAlarmCondition threw for {Node}", alarmNodeId); }
+        try { _sink.MaterialiseAlarmCondition(alarmNodeId, equipmentNodeId, displayName, alarmType, severity, isNative); return true; }
+        catch (Exception ex) { _logger.LogWarning(ex, "AddressSpaceApplier: MaterialiseAlarmCondition threw for {Node}", alarmNodeId); return false; }
     }
 }
 
-/// <summary>Summary of one apply pass. Useful for tests + audit-log entries on the deploy path.</summary>
+/// <summary>Summary of one apply pass. Useful for tests + audit-log entries on the deploy path.
+/// <para><see cref="RebuildCalled"/> means a rebuild was ATTEMPTED; <see cref="RebuildFailed"/> means the
+/// attempt threw (the sink's address space is now stale/partial). <see cref="FailedNodes"/> counts
+/// swallowed per-node sink failures in <see cref="AddressSpaceApplier.Apply"/>'s own passes (the removal
+/// alarm-condition writes). The <c>Materialise*</c> passes report their own failed-node tallies via their
+/// <c>int</c> returns — the publish actor sums both channels so a degraded apply is operator-visible
+/// (Error log + <c>otopcua.opcua.apply.failed</c> meter) instead of reported as optimistic success
+/// (archreview 01/S-1). New trailing fields are defaulted so every existing construction compiles.</para></summary>
 public sealed record AddressSpaceApplyOutcome(
     int RemovedNodes,
     int AddedNodes,
     int ChangedNodes,
-    bool RebuildCalled);
+    bool RebuildCalled,
+    bool RebuildFailed = false,
+    int FailedNodes = 0);

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs

@@ -192,14 +192,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
 
     /// <summary>
     ///     Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
-    ///     snapshot (null = unknown until the first snapshot arrives, or no local node match). The inbound
-    ///     write gate in <see cref="HandleRouteNodeWrite"/> reuses this signal — the SAME one the
-    ///     scripted-alarm emit gate uses (<c>ScriptedAlarmHostActor._localRole</c>): only the Primary
-    ///     services writes, default-allow while unknown so single-node deploys + the boot window never
-    ///     reject (a node is the sole Primary until told otherwise).
+    ///     snapshot (null = unknown until the first snapshot arrives, or no local node match). The Primary
+    ///     data-plane gates (<see cref="HandleRouteNodeWrite"/>, <see cref="HandleRouteNativeAlarmAck"/>,
+    ///     <see cref="ForwardNativeAlarm"/>) resolve this through <see cref="PrimaryGatePolicy"/>: a KNOWN
+    ///     role wins outright; an UNKNOWN role is resolved by cluster membership — a single-driver cluster
+    ///     stays default-ALLOW (boot-window / single-node), a multi-driver cluster default-DENIES until a
+    ///     snapshot proves this node is Primary (archreview 03/S4 — closes the dual-primary boot window).
     /// </summary>
     private RedundancyRole? _localRole;
 
+    /// <summary>Test seam (archreview 03/S4): overrides the count of Up <c>driver</c>-role cluster members
+    /// the Primary gate reads while the role is unknown. Null ⇒ read the live cluster state (0 on a
+    /// non-cluster ActorRefProvider). See <see cref="DriverMemberCount"/>.</summary>
+    private readonly Func<int>? _driverMemberCountProvider;
+
+    /// <summary>Debounces the S5 "snapshot omitted this node" Warning to once per process — a persistent
+    /// identity mismatch (03/S5) would otherwise log on every snapshot.</summary>
+    private bool _warnedSnapshotMissingLocalNode;
+
     /// <summary>Cached cluster DistributedPubSub mediator, resolved once in <see cref="PreStart"/> (on the
     /// actor thread) and reused for the Primary-gated native-alarm <c>alerts</c> fan-out in
     /// <see cref="ForwardNativeAlarm"/> instead of re-resolving it per-publish. Mirrors
@@ -308,11 +318,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         ILoggerFactory? loggerFactory = null,
         ScriptRootLogger? scriptRootLogger = null,
         IActorRef? scriptedAlarmHostOverride = null,
-        IDriverCapabilityInvokerFactory? invokerFactory = null) =>
+        IDriverCapabilityInvokerFactory? invokerFactory = null,
+        Func<int>? driverMemberCountProvider = null) =>
         Akka.Actor.Props.Create(() => new DriverHostActor(
             dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
             healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
-            loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory));
+            loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider));
 
     /// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
     /// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -338,6 +349,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
     /// <param name="invokerFactory">Phase 6.1 resilience-invoker factory used to attach an
     /// <see cref="IDriverCapabilityInvoker"/> to each spawned driver; defaults to
     /// <see cref="NullDriverCapabilityInvokerFactory"/> (pass-through) when null.</param>
+    /// <param name="driverMemberCountProvider">Test seam (archreview 03/S4): overrides the count of Up
+    /// <c>driver</c>-role cluster members the Primary gate reads while the role is unknown. When null the
+    /// default reads <c>Cluster.Get(Context.System).State.Members</c> (0 on a non-cluster ActorRefProvider).</param>
     public DriverHostActor(
         IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
         CommonsNodeId localNode,
@@ -353,13 +367,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         ILoggerFactory? loggerFactory = null,
         ScriptRootLogger? scriptRootLogger = null,
         IActorRef? scriptedAlarmHostOverride = null,
-        IDriverCapabilityInvokerFactory? invokerFactory = null)
+        IDriverCapabilityInvokerFactory? invokerFactory = null,
+        Func<int>? driverMemberCountProvider = null)
     {
         _dbFactory = dbFactory;
         _localNode = localNode;
         _coordinatorOverride = coordinator;
         _driverFactory = driverFactory ?? NullDriverFactory.Instance;
         _invokerFactory = invokerFactory ?? NullDriverCapabilityInvokerFactory.Instance;
+        _driverMemberCountProvider = driverMemberCountProvider;
         _localRoles = localRoles ?? new HashSet<string>(StringComparer.Ordinal);
         _dependencyMux = dependencyMux;
         _opcUaPublishActor = opcUaPublishActor;
@@ -964,17 +980,29 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
                 _localNode, msg.DriverInstanceId, msg.Args.ConditionId);
             return;
         }
+        // PRIMARY GATE (archreview 03/S4) — decided ONCE for this transition: the OPC UA condition write
+        // below stays UNGATED (a Secondary keeps its address space warm for failover), but only the Primary
+        // publishes the single cluster-wide alerts copy. Denied on a Secondary/Detached, OR on a multi-driver
+        // cluster while the role is unknown (default-deny closes the dual-primary boot window that would
+        // otherwise historize duplicate AVEVA alarm rows). Metered + Debug-logged once when denied.
+        var serviceAlertsAsPrimary = ShouldServiceAsPrimary();
+        if (!serviceAlertsAsPrimary)
+        {
+            OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
+                new KeyValuePair<string, object?>("site", "alarm-emit"),
+                new KeyValuePair<string, object?>("reason", PrimaryGateDenyReason()));
+            _log.Debug("DriverHost {Node}: native-alarm alerts publish suppressed for ({Driver},{Ref}) — not primary",
+                _localNode, msg.DriverInstanceId, msg.Args.ConditionId);
+        }
+
         foreach (var nodeId in nodeIds)
         {
             var snapshot = _nativeAlarmProjector.Project(nodeId, msg.Args);
             _opcUaPublishActor.Tell(new ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.AlarmStateUpdate(
                 nodeId, snapshot, msg.Args.SourceTimestampUtc));
 
-            // Warm-standby dedup: the OPC UA condition write above is UNGATED (the secondary keeps its
-            // address space warm), but only the Primary publishes the cluster-wide alerts transition.
-            // Default-emit until told we are Secondary/Detached so single-node deploys + the boot window
-            // never drop transitions — the SAME signal HandleRouteNodeWrite gates writes on.
-            if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached) continue;
+            // Only the Primary publishes the cluster-wide alerts transition (see the gate above).
+            if (!serviceAlertsAsPrimary) continue;
 
             var meta = _alarmMetaByNodeId.TryGetValue(nodeId, out var m)
                 ? m : (EquipmentId: nodeId, Name: nodeId, AlarmType: "AlarmCondition", HistorizeToAveva: (bool?)null);
@@ -1034,11 +1062,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
     /// </summary>
     private void HandleRouteNodeWrite(RouteNodeWrite msg)
     {
-        // PRIMARY GATE FIRST — only the Primary services operator writes (same signal as the alarm-emit
-        // gate; unknown role ⇒ treated as Primary so single-node deploys + the boot window aren't blocked).
-        if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached)
+        // PRIMARY GATE FIRST (archreview 03/S4) — only the Primary services operator writes. A KNOWN
+        // Secondary/Detached is denied "not primary" (unchanged); an UNKNOWN role denies ONLY on a
+        // multi-driver cluster ("not primary (role unknown)"), while a single-driver/boot-window cluster is
+        // still serviced. The rejection rides the established optimistic-write self-correction surface (the
+        // node reverts to its prior value with a Bad blip + a Part 8 audit event) — writes are NOT queued.
+        if (!ShouldServiceAsPrimary())
         {
-            Sender.Tell(new NodeWriteResult(false, "not primary"));
+            var reason = _localRole is null ? "not primary (role unknown)" : "not primary";
+            OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
+                new KeyValuePair<string, object?>("site", "write"),
+                new KeyValuePair<string, object?>("reason", PrimaryGateDenyReason()));
+            Sender.Tell(new NodeWriteResult(false, reason));
             return;
         }
 
@@ -1090,12 +1125,25 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
     /// </summary>
     private void HandleRouteNativeAlarmAck(RouteNativeAlarmAck msg)
     {
-        // PRIMARY GATE FIRST — only the Primary services operator acks (same signal as the inbound-write +
-        // alarm-emit gates; unknown role ⇒ treated as Primary so single-node deploys + the boot window aren't blocked).
-        if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached)
+        // PRIMARY GATE FIRST (archreview 03/S4) — only the Primary pushes the ack upstream. Same policy as
+        // the inbound-write gate. A KNOWN Secondary/Detached is dropped at Debug (unchanged); an UNKNOWN role
+        // on a multi-driver cluster is dropped at WARNING (an operator ack silently not reaching the upstream
+        // alarm system is a swallowed failure worth surfacing).
+        if (!ShouldServiceAsPrimary())
         {
-            _log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary",
-                _localNode, msg.ConditionNodeId);
+            OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
+                new KeyValuePair<string, object?>("site", "ack"),
+                new KeyValuePair<string, object?>("reason", PrimaryGateDenyReason()));
+            if (_localRole is null)
+            {
+                _log.Warning("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary (role unknown on multi-driver cluster)",
+                    _localNode, msg.ConditionNodeId);
+            }
+            else
+            {
+                _log.Debug("DriverHost {Node}: dropping native-alarm ack for {Cond} — not primary",
+                    _localNode, msg.ConditionNodeId);
+            }
             return;
         }
 
@@ -1119,9 +1167,43 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         entry.Actor.Tell(new DriverInstanceActor.RouteAlarmAck(target.FullName, msg.Comment, msg.OperatorUser));
     }
 
-    /// <summary>Caches this node's <see cref="RedundancyRole"/> from a cluster redundancy snapshot so
-    /// <see cref="HandleRouteNodeWrite"/> can gate inbound writes to the Primary. A snapshot that doesn't
-    /// mention this node leaves the cached role unchanged ⇒ default-allow. Mirrors
+    /// <summary>Whether this node should service a Primary-only data-plane operation right now — the single
+    /// gate decision (archreview 03/S4), delegating to <see cref="PrimaryGatePolicy"/> with the live driver
+    /// member count.</summary>
+    /// <returns><c>true</c> to service as Primary; <c>false</c> to deny.</returns>
+    private bool ShouldServiceAsPrimary() =>
+        PrimaryGatePolicy.ShouldServiceAsPrimary(_localRole, DriverMemberCount());
+
+    /// <summary>Count of Up cluster members carrying the <c>driver</c> role. Uses the injected test seam when
+    /// present; otherwise reads the live cluster state guarded by try/catch so a non-cluster
+    /// ActorRefProvider (legacy/test harnesses) yields 0 ⇒ the single-node default-allow posture.</summary>
+    /// <returns>The number of Up driver-role members, or 0 when the cluster extension is unavailable.</returns>
+    private int DriverMemberCount()
+    {
+        if (_driverMemberCountProvider is not null) return _driverMemberCountProvider();
+        try
+        {
+            return Akka.Cluster.Cluster.Get(Context.System).State.Members
+                .Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole));
+        }
+        catch (Exception)
+        {
+            return 0;   // non-cluster ActorRefProvider (legacy harnesses) ⇒ single-node posture
+        }
+    }
+
+    /// <summary>The Primary-gate deny reason tag for the denial meter (<c>secondary|detached|role-unknown</c>).</summary>
+    private string PrimaryGateDenyReason() => _localRole switch
+    {
+        RedundancyRole.Secondary => "secondary",
+        RedundancyRole.Detached => "detached",
+        _ => "role-unknown",
+    };
+
+    /// <summary>Caches this node's <see cref="RedundancyRole"/> from a cluster redundancy snapshot so the
+    /// Primary data-plane gates can resolve their decision. A snapshot that doesn't mention this node leaves
+    /// the cached role unchanged; when this node has a real driver peer (count &gt; 1) that omission is the
+    /// 03/S5 identity-mismatch shape, logged ONCE at Warning. Mirrors
     /// <c>ScriptedAlarmHostActor.OnRedundancyStateChanged</c> / <c>OpcUaPublishActor</c>.</summary>
     private void OnRedundancyStateChanged(RedundancyStateChanged msg)
     {
@@ -1129,6 +1211,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
         if (local is not null)
         {
             _localRole = local.Role;
+            return;
+        }
+
+        // S5 diagnostic: the snapshot never mentioned this node. On a multi-driver cluster that means the
+        // Primary gate stays default-DENY indefinitely for an unknown role — surface it once so a silent
+        // identity mismatch (e.g. an ApplicationUri/NodeId skew) is diagnosable.
+        if (!_warnedSnapshotMissingLocalNode && DriverMemberCount() > 1)
+        {
+            _warnedSnapshotMissingLocalNode = true;
+            _log.Warning(
+                "DriverHost {Node}: redundancy snapshot omitted this node (snapshotNodes={SnapshotNodes}); Primary data-plane gate stays default-DENY while role is unknown on a multi-driver cluster",
+                _localNode, string.Join(",", msg.Nodes.Select(n => n.NodeId)));
         }
     }
 

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/PrimaryGatePolicy.cs

@@ -0,0 +1,28 @@
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
+
+/// <summary>
+///     Single decision point for the Primary data-plane gates (inbound device writes, native alarm acks,
+///     fleet-wide alerts publishes). A KNOWN role wins outright; an UNKNOWN role (no redundancy snapshot
+///     yet, or the snapshot never mentioned this node — the 03/S5 identity-mismatch shape) is resolved by
+///     cluster membership: a single-driver cluster stays default-ALLOW (no peer exists — the
+///     boot-window / single-node posture), a multi-driver cluster is default-DENY (a real Primary peer
+///     exists; do not touch the shared field device until the redundancy snapshot proves this node is it).
+///     Fixes archreview 03/S4 (primary gate defaulted to allow while role unknown → dual-primary window).
+/// </summary>
+public static class PrimaryGatePolicy
+{
+    /// <summary>Decide whether this node should service a Primary-only data-plane operation.</summary>
+    /// <param name="localRole">This node's last-known redundancy role, or <c>null</c> when unknown
+    /// (no snapshot received yet, or the snapshot omitted this node).</param>
+    /// <param name="driverMemberCount">Count of Up cluster members carrying the <c>driver</c> role.</param>
+    /// <returns><c>true</c> to service as Primary; <c>false</c> to deny.</returns>
+    public static bool ShouldServiceAsPrimary(RedundancyRole? localRole, int driverMemberCount) =>
+        localRole switch
+        {
+            RedundancyRole.Primary => true,
+            RedundancyRole.Secondary or RedundancyRole.Detached => false,
+            _ => driverMemberCount <= 1,   // unknown: allow only when no driver peer exists
+        };
+}

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/OpcUa/OpcUaPublishActor.cs

@@ -335,27 +335,43 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
             var outcome = _applier.Apply(plan);
             _lastApplied = composition;
 
-            _applier.MaterialiseHierarchy(composition);
+            // Sum swallowed per-node materialise failures across every pass together with Apply's own
+            // removal-pass tally (archreview 01/S-1). A degraded apply used to vanish into per-node
+            // Warnings + an optimistic Info line; now it surfaces at Error + a dedicated meter.
+            var failedNodes = outcome.FailedNodes;
+            failedNodes += _applier.MaterialiseHierarchy(composition);
             // T14 — scripted alarms get their own pass right after the hierarchy so the equipment
             // folders they parent under already exist. Materialises real Part 9 AlarmConditionState
             // nodes (keyed by ScriptedAlarmId so AlarmStateUpdate writes target them); disabled
             // alarms are skipped.
-            _applier.MaterialiseScriptedAlarms(composition);
+            failedNodes += _applier.MaterialiseScriptedAlarms(composition);
             // Equipment-namespace tags get their own pass: ensures each signal's Variable (and any
             // FolderPath sub-folder) exists under its already-materialised equipment folder so
             // clients can browse them. Live values are pushed by DriverHostActor.ForwardToMux after
             // each subscription cycle; variables show BadWaitingForInitialData only until the first
             // publish interval fires.
-            _applier.MaterialiseEquipmentTags(composition);
+            failedNodes += _applier.MaterialiseEquipmentTags(composition);
             // Equipment-namespace VirtualTags get their own pass right after the equipment tags:
             // ensures each computed signal's Variable (and any FolderPath sub-folder) exists under its
             // equipment folder with a folder-scoped NodeId. VirtualTagHostActor.OnResult pushes live
             // values once the first dependency update arrives; until then variables show BadWaitingForInitialData.
-            _applier.MaterialiseEquipmentVirtualTags(composition);
+            failedNodes += _applier.MaterialiseEquipmentVirtualTags(composition);
 
             OtOpcUaTelemetry.OpcUaSinkWrite.Add(1, new KeyValuePair<string, object?>("kind", "rebuild"));
-            _log.Info("OpcUaPublish: applied rebuild (correlation={Correlation}, added={Added}, removed={Removed}, changed={Changed}, rebuild={Rebuild})",
-                msg.Correlation, outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes, outcome.RebuildCalled);
+            if (outcome.RebuildFailed || failedNodes > 0)
+            {
+                OtOpcUaTelemetry.OpcUaApplyFailed.Add(1,
+                    new KeyValuePair<string, object?>("kind", outcome.RebuildFailed ? "rebuild" : "nodes"));
+                _log.Error(
+                    "OpcUaPublish: address-space apply DEGRADED (correlation={Correlation}, rebuildFailed={RebuildFailed}, failedNodes={FailedNodes}, added={Added}, removed={Removed}, changed={Changed}) — the running address space may be stale or partial",
+                    msg.Correlation, outcome.RebuildFailed, failedNodes,
+                    outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes);
+            }
+            else
+            {
+                _log.Info("OpcUaPublish: applied rebuild (correlation={Correlation}, added={Added}, removed={Removed}, changed={Changed}, rebuild={Rebuild})",
+                    msg.Correlation, outcome.AddedNodes, outcome.RemovedNodes, outcome.ChangedNodes, outcome.RebuildCalled);
+            }
         }
         catch (Exception ex)
         {
@@ -412,7 +428,16 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
             _log.Debug("OpcUaPublish: no applier wired — discarding MaterialiseDiscoveredNodes for {Equipment}", msg.EquipmentRootNodeId);
             return;
         }
-        _applier.MaterialiseDiscoveredNodes(msg.EquipmentRootNodeId, msg.Folders, msg.Variables);
+        var failedNodes = _applier.MaterialiseDiscoveredNodes(msg.EquipmentRootNodeId, msg.Folders, msg.Variables);
+        if (failedNodes > 0)
+        {
+            // archreview 01/S-1: a swallowed discovered-node injection failure surfaces at Error + the
+            // dedicated meter instead of vanishing into per-node Warnings.
+            OtOpcUaTelemetry.OpcUaApplyFailed.Add(1, new KeyValuePair<string, object?>("kind", "nodes"));
+            _log.Error(
+                "OpcUaPublish: discovered-node injection DEGRADED for {Equipment} (failedNodes={FailedNodes}) — some discovered nodes may be missing",
+                msg.EquipmentRootNodeId, failedNodes);
+        }
     }
 
     private void HandleServiceLevelChanged(ServiceLevelChanged msg)

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs

@@ -3,11 +3,13 @@ using Akka.Cluster.Tools.PublishSubscribe;
 using Akka.Event;
 using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
 using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
+using ZB.MOM.WW.OtOpcUa.Commons.Observability;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Commons.Types;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
 using ZB.MOM.WW.OtOpcUa.OpcUaServer;
+using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
 using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
 
@@ -104,10 +106,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
 
     /// <summary>Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
     /// snapshot (null = unknown until the first snapshot arrives, or no <see cref="_localNode"/> wired). The
-    /// cluster-wide <c>alerts</c> publish in <see cref="OnEngineEmission"/> is gated on this: only the Primary
-    /// publishes (default-emit while unknown so single-node deploys + the boot window never drop transitions).</summary>
+    /// cluster-wide <c>alerts</c> publish in <see cref="OnEngineEmission"/> resolves this through
+    /// <see cref="Drivers.PrimaryGatePolicy"/>: only the Primary publishes; an unknown role default-emits on a
+    /// single-driver cluster but default-DENIES on a multi-driver one, so the dual-primary boot window can't
+    /// historize duplicate AVEVA alarm rows (archreview 03/S4).</summary>
     private RedundancyRole? _localRole;
 
+    /// <summary>Test seam (archreview 03/S4): overrides the count of Up <c>driver</c>-role cluster members the
+    /// alerts-emit gate reads while the role is unknown. Null ⇒ read the live cluster state.</summary>
+    private readonly Func<int>? _driverMemberCountProvider;
+
     /// <summary>Monotonic load generation, bumped on every <see cref="OnApply"/>. The continuation that
     /// pipes back an <see cref="AlarmsLoaded"/> captures the generation it was started under; a stale
     /// completion (an earlier generation arriving after a newer apply) is discarded in
@@ -136,8 +144,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
         IActorRef? mux,
         DependencyMuxTagUpstreamSource upstream,
         ScriptedAlarmEngine engine,
-        NodeId? localNode = null) =>
-        Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode));
+        NodeId? localNode = null,
+        Func<int>? driverMemberCountProvider = null) =>
+        Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider));
 
     /// <summary>Initializes a new instance of the <see cref="ScriptedAlarmHostActor"/> class.</summary>
     /// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
@@ -147,12 +156,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
     /// <param name="localNode">The local cluster node id, used to read this node's <see cref="RedundancyRole"/>
     /// from the <c>redundancy-state</c> topic so only the Primary publishes the cluster-wide <c>alerts</c>
     /// transition. Null leaves the role unknown ⇒ default-emit (single-node deploys + tests).</param>
+    /// <param name="driverMemberCountProvider">Test seam (archreview 03/S4): overrides the count of Up
+    /// <c>driver</c>-role cluster members the alerts-emit gate reads while the role is unknown. Null reads
+    /// the live cluster state (0 on a non-cluster ActorRefProvider ⇒ single-node default-emit).</param>
     public ScriptedAlarmHostActor(
         IActorRef publishActor,
         IActorRef? mux,
         DependencyMuxTagUpstreamSource upstream,
         ScriptedAlarmEngine engine,
-        NodeId? localNode = null)
+        NodeId? localNode = null,
+        Func<int>? driverMemberCountProvider = null)
     {
         ArgumentNullException.ThrowIfNull(publishActor);
         ArgumentNullException.ThrowIfNull(upstream);
@@ -162,6 +175,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
         _upstream = upstream;
         _engine = engine;
         _localNode = localNode;
+        _driverMemberCountProvider = driverMemberCountProvider;
 
         // OnEvent fires on the engine's worker thread. NEVER touch Context / actor state here —
         // marshal onto the actor thread via the thread-safe Self.Tell. Keep the handler in a field
@@ -307,18 +321,45 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
             // suppress only the sink write. This publish (and the live `/alerts` fan-out) is NOT gated on it.
             HistorizeToAveva: e.HistorizeToAveva);
 
-        // Warm-standby dedup: only the Primary (driver-role leader) publishes the cluster-wide
-        // transition. Default-emit until told we are Secondary/Detached so single-node deploys + the
-        // boot window never drop transitions. The OPC UA node write above + inbound command processing
-        // stay ungated (the secondary keeps its address space + engine state warm for failover).
-        if (_localRole is RedundancyRole.Secondary or RedundancyRole.Detached)
+        // Warm-standby dedup: only the Primary publishes the cluster-wide transition (archreview 03/S4 —
+        // same PrimaryGatePolicy as DriverHostActor). A KNOWN Secondary/Detached is denied; an UNKNOWN role
+        // default-emits on a single-driver cluster but default-DENIES on a multi-driver one (so the
+        // dual-primary boot window can't historize duplicate AVEVA alarm rows). The OPC UA node write above +
+        // inbound command processing stay ungated (the secondary keeps its state warm for failover).
+        if (!PrimaryGatePolicy.ShouldServiceAsPrimary(_localRole, DriverMemberCount()))
         {
+            OtOpcUaTelemetry.PrimaryGateDenied.Add(1,
+                new KeyValuePair<string, object?>("site", "alarm-emit"),
+                new KeyValuePair<string, object?>("reason", _localRole switch
+                {
+                    RedundancyRole.Secondary => "secondary",
+                    RedundancyRole.Detached => "detached",
+                    _ => "role-unknown",
+                }));
             return;
         }
 
         _mediator.Tell(new Publish(AlertsTopic, evt));
     }
 
+    /// <summary>Count of Up cluster members carrying the <c>driver</c> role, for the alerts-emit gate. Uses
+    /// the injected test seam when present; otherwise reads the live cluster state guarded by try/catch so a
+    /// non-cluster ActorRefProvider yields 0 ⇒ the single-node default-emit posture (archreview 03/S4).</summary>
+    /// <returns>The number of Up driver-role members, or 0 when the cluster extension is unavailable.</returns>
+    private int DriverMemberCount()
+    {
+        if (_driverMemberCountProvider is not null) return _driverMemberCountProvider();
+        try
+        {
+            return Akka.Cluster.Cluster.Get(Context.System).State.Members
+                .Count(m => m.Status == Akka.Cluster.MemberStatus.Up && m.Roles.Contains(ServiceCollectionExtensions.DriverRole));
+        }
+        catch (Exception)
+        {
+            return 0;
+        }
+    }
+
     /// <summary>Caches this node's <see cref="RedundancyRole"/> from a cluster redundancy snapshot so
     /// <see cref="OnEngineEmission"/> can gate the cluster-wide <c>alerts</c> publish to the Primary. A
     /// snapshot that doesn't mention <see cref="_localNode"/> (or no local node wired) leaves the cached