lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/WarmRedundancyWireTests.cs

using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig;

namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;

/// <summary>
///     Gitea #717 over the wire, against the real <see cref="OtOpcUaSdkServer"/> and node manager: the redundancy
///     metadata per mode (T2), the Standby read projection and write refusal (T6), the D4 first-delivery decision record
///     across the three demotion timings T0 established (T7), the live-item queue on promotion/demotion (T8), and the
///     alarm suppression, refusal and promotion refresh (T9). Transitions are driven through the server's own
///     <see cref="IStandbyProjection"/> in the spec §4.7 step order; the actor that owns that order has its own tests.
/// </summary>
[Trait("Category", "Integration")]
public sealed partial class WarmRedundancyWireTests
{
    private const string Peer = "urn:OtOpcUa:peer-717";

    private const NodePermissions All =
        NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead
        | NodePermissions.AlarmAcknowledge | NodePermissions.AlarmConfirm | NodePermissions.AlarmShelve
        | NodePermissions.MethodCall | NodePermissions.WriteOperate;

    internal static Task<ServerEventRig> StartModeAsync(string name, string mode, CancellationToken ct) =>
        ServerEventRig.StartAsync(name, encrypted: false, ct, o =>
        {
            o.Redundancy = new OpcUaRedundancyOptions { Mode = mode };
            if (mode == "Warm") o.PeerApplicationUris = [Peer];
        });

    /// <summary>Spec §4.7 promotion, in order (the actor's sequence, minus the ServiceLevel publish).</summary>
    internal static void Promote(IStandbyProjection p)
    {
        p.SetDataProjection(false);
        p.QueueCurrentToLiveItems(false);
        p.SetStandbyDelivery(false);
        p.SetAlarmSuppression(false);
        p.RefreshAllEventItems();
    }

    /// <summary>Spec §4.7 demotion, in order (after the Standby ServiceLevel publish).</summary>
    internal static void Demote(IStandbyProjection p)
    {
        p.SetStandbyDelivery(true);
        p.SetAlarmSuppression(true);
        p.SetDataProjection(true);
        p.QueueCurrentToLiveItems(true);
    }

    internal static CancellationToken Deadline(int seconds = 90)
    {
        var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
        cts.CancelAfter(TimeSpan.FromSeconds(seconds));
        return cts.Token;
    }

    internal static RequestHeader H() => new() { Timestamp = DateTimeUtc.Now, TimeoutHint = 10_000 };

    internal static NodeId Raw(ISession session, string id) =>
        new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri));

    internal static async Task<DataValue> ReadAsync(ISession session, NodeId node, CancellationToken ct, uint attribute = Attributes.Value,
        string? indexRange = null)
    {
        var r = await ((SessionClient)session).ReadAsync(H(), 0, TimestampsToReturn.Both,
            new[] { new ReadValueId { NodeId = node, AttributeId = attribute, IndexRange = indexRange } }, ct);
        return r.Results[0];
    }

    // ------------------------------------------------------------------ T2 metadata

    [Fact]
    public async Task Warm_advertises_non_transparent_metadata_self_first_and_boots_at_ServiceLevel_1()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717Meta", "Warm", ct);
        using var session = await rig.ConnectAsync(anonymous: true, ct);

        Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue())
            .ShouldBe((int)RedundancySupport.Warm);
        var uris = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct);
        uris.StatusCode.ShouldBe(StatusCodes.Good);
        ((string[])uris.ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer });
        ((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri, Peer });
        Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.WarmBootServiceLevel);

        var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[]
        {
            new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward,
                ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All },
        }, ct);
        browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.NonTransparentRedundancyType));

        var write = await ((SessionClient)session).WriteAsync(H(), new[]
        {
            new WriteValue { NodeId = VariableIds.Server_ServerRedundancy_ServerUriArray, AttributeId = Attributes.Value,
                Value = new DataValue(new Variant(new[] { "x" })) },
        }, ct);
        StatusCode.IsBad(write.Results[0]).ShouldBeTrue($"ServerUriArray is read-only, got {write.Results[0]}");
    }

    [Fact]
    public async Task None_advertises_RedundancySupport_None_and_ServerUriArray_is_absent_even_by_NodeId()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717None", "None", ct);
        using var session = await rig.ConnectAsync(anonymous: true, ct);

        Convert.ToInt32((await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).ClrValue())
            .ShouldBe((int)RedundancySupport.None);
        (await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct)).StatusCode
            .ShouldBe(StatusCodes.BadNodeIdUnknown, "T0a: the SDK's orphan i=11314 is deleted under None");
        var browse = await ((SessionClient)session).BrowseAsync(H(), null, 0, new[]
        {
            new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward,
                ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All },
        }, ct);
        browse.Results[0].References[0].NodeId.ShouldBe(new ExpandedNodeId(ObjectTypeIds.ServerRedundancyType));
        ((string[])(await ReadAsync(session, VariableIds.Server_ServerArray, ct)).ClrValue()!).ShouldBe(new[] { rig.ApplicationUri });
    }

    /// <summary>The default-off pin: an Unset server is today's server — SDK-default RedundancySupport, the SDK's own
    /// (orphan) i=11314, ServerRedundancyType, the #885 boot level 0, Good data with no projection, and no Warm hooks.</summary>
    [Fact]
    public async Task Unset_is_todays_server_exactly()
    {
        var ct = Deadline();
        await using var rig = await ServerEventRig.StartAsync("Warm717Unset", encrypted: false, ct);
        rig.Server.RedundancyMode.ShouldBe(OpcUaRedundancyMode.Unset);
        rig.Server.CreateStandbyProjection().ShouldBeNull();
        rig.Nm.Standby.WarmMode.ShouldBeFalse();
        var id = rig.EnsureRawVariable("u", "Double", writable: false);
        rig.Sink.WriteValue(id, 4.5, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        var support = await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct);
        support.StatusCode.ShouldBe(StatusCodes.Good);
        Convert.ToInt32(support.ClrValue()).ShouldBe((int)RedundancySupport.None, "the SDK default, never written by us");
        var orphan = await ReadAsync(session, VariableIds.Server_ServerRedundancy_ServerUriArray, ct);
        orphan.StatusCode.ShouldBe(StatusCodes.Good, "Unset leaves the SDK's orphan i=11314 exactly as the SDK builds it");
        Convert.ToByte((await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).ClrValue()).ShouldBe(OtOpcUaSdkServer.StartupServiceLevel);
        var value = await ReadAsync(session, Raw(session, id), ct);
        value.StatusCode.ShouldBe(StatusCodes.Good);
        value.ClrValue().ShouldBe(4.5);
    }

    // ------------------------------------------------------------------ T6 read projection + write refusal

    [Fact]
    public async Task Standby_projects_ordinary_tag_reads_but_not_metadata_and_promotion_reads_the_latest_stored_value()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717Read", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection().ShouldNotBeNull();
        var raw = rig.EnsureRawVariable("t1", "Double", writable: false);
        rig.Sink.EnsureVariable("EqA/t1ref", null, "t1ref", "Double", writable: false, AddressSpaceRealm.Uns);
        var arr = $"{RawDeviceFolder}/arr";
        rig.Sink.EnsureVariable(arr, RawDeviceFolder, "arr", "Int32", writable: false, AddressSpaceRealm.Raw, isArray: true);
        rig.EnsureEquipment();
        rig.Sink.EnsureVariable("EqA/vt", "EqA", "vt", "Double", writable: false, AddressSpaceRealm.Uns);
        rig.MaterialiseScripted();
        var ts = DateTime.UtcNow;
        rig.Sink.WriteValue(raw, 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw);
        rig.Sink.WriteValue("EqA/t1ref", 1.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns);
        rig.Sink.WriteValue("EqA/vt", 2.5, OpcUaStatus.Good, ts, AddressSpaceRealm.Uns);
        rig.Sink.WriteValue(arr, new[] { 1, 2, 3 }, OpcUaStatus.Good, ts, AddressSpaceRealm.Raw);

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        foreach (var node in new[] { Raw(session, raw), UnsNodeId(session, "EqA/t1ref"), UnsNodeId(session, "EqA/vt") })
        {
            var dv = await ReadAsync(session, node, ct);
            dv.StatusCode.ShouldBe(StatusCodes.BadOutOfService, node.ToString());
            dv.WrappedValue.IsNull.ShouldBeTrue();
            (dv.ServerTimestamp != DateTimeUtc.MinValue).ShouldBeTrue();
        }

        (await ReadAsync(session, Raw(session, arr), ct, indexRange: "1")).StatusCode.ShouldBe(StatusCodes.BadOutOfService);
        (await ReadAsync(session, Raw(session, raw), ct, Attributes.DisplayName)).StatusCode.ShouldBe(StatusCodes.Good);
        (await ReadAsync(session, VariableIds.Server_ServiceLevel, ct)).StatusCode.ShouldBe(StatusCodes.Good);
        (await ReadAsync(session, VariableIds.Server_ServerRedundancy_RedundancySupport, ct)).StatusCode.ShouldBe(StatusCodes.Good);
        var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct);
        (await ReadAsync(session, severity, ct)).StatusCode.ShouldBe(StatusCodes.Good, "a condition child is not an ordinary tag");

        // Acquisition continues while projected; promotion reads the latest stored value with no new driver publish.
        rig.Sink.WriteValue(raw, 9.25, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        Promote(projection);
        var after = await ReadAsync(session, Raw(session, raw), ct);
        after.StatusCode.ShouldBe(StatusCodes.Good);
        after.ClrValue().ShouldBe(9.25);
        var ranged = await ReadAsync(session, Raw(session, arr), ct, indexRange: "1");
        ranged.StatusCode.ShouldBe(StatusCodes.Good);
        ((int[])ranged.ClrValue()!).ShouldBe(new[] { 2 }, "an IndexRange read reaches the stored array once promoted");
    }

    [Fact]
    public async Task Standby_projection_never_leaks_an_ACL_denied_node()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717Acl", "Warm", ct);
        var raw = rig.EnsureRawVariable("secret", "Double", writable: false);
        rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        rig.Enforce(true, [Key()]); // anonymous holds no grant

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        (await ReadAsync(session, Raw(session, raw), ct)).StatusCode.ShouldBe(StatusCodes.BadUserAccessDenied);
    }

    private sealed class RecordingWriteGateway : IOpcUaNodeWriteGateway
    {
        public int Count;

        public Task<NodeWriteOutcome> WriteAsync(string nodeId, object? value, AddressSpaceRealm realm, CancellationToken ct)
        {
            Interlocked.Increment(ref Count);
            return Task.FromResult(new NodeWriteOutcome(true, null));
        }
    }

    [Theory]
    [InlineData("Warm", true)]
    [InlineData("Warm", false)]
    [InlineData("Unset", true)]
    public async Task A_write_on_a_Warm_standby_is_refused_BadInvalidState_and_never_reaches_the_gateway(string mode, bool standby)
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync($"Warm717Write{mode}{standby}", mode, ct);
        var gateway = new RecordingWriteGateway();
        rig.Nm.NodeWriteGateway = gateway;
        rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
        var raw = rig.EnsureRawVariable("w", "Double", writable: true);
        rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        if (mode == "Warm" && !standby) Promote(rig.Server.CreateStandbyProjection()!);
        // Unset with "standby": a node that is not the Primary takes today's optimistic path (no gate wired here).

        using var session = await rig.ConnectAsync(anonymous: false, ct);
        var result = await ((SessionClient)session).WriteAsync(H(), new[]
        {
            new WriteValue { NodeId = Raw(session, raw), AttributeId = Attributes.Value, Value = new DataValue(new Variant(2.0)) },
        }, ct);

        if (mode == "Warm" && standby)
        {
            result.Results[0].ShouldBe(StatusCodes.BadInvalidState);
            await Task.Delay(200, ct);
            gateway.Count.ShouldBe(0, "a refused write never reaches the driver");
        }
        else
        {
            result.Results[0].ShouldBe(StatusCodes.Good);
            await RecordedEvents.WaitUntilAsync(() => Volatile.Read(ref gateway.Count) == 1);
        }
    }

    // ------------------------------------------------------------------ T9 Part 9 method refusal

    [Theory]
    [InlineData("Warm", true, "Acknowledge")]
    [InlineData("Warm", true, "OneShotShelve")]
    [InlineData("Warm", false, "Acknowledge")]
    [InlineData("Unset", true, "Acknowledge")]
    public async Task A_Part9_method_on_a_Warm_standby_is_refused_BadInvalidState_before_any_state_change(
        string mode, bool standby, string method)
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync($"Warm717Method{mode}{standby}{method}", mode, ct);
        rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
        rig.MaterialiseScripted();
        var routed = 0;
        rig.Nm.AlarmCommandRouter = _ => Interlocked.Increment(ref routed);
        var projection = rig.Server.CreateStandbyProjection();
        if (projection is not null) Promote(projection); // project the condition while Active so it has an EventId
        rig.Project(native: false, ConditionMethodEventTests.PreState(method, native: false) with { Message = "base" });
        if (projection is not null && standby) Demote(projection);

        using var session = await rig.ConnectAsync(anonymous: false, ct);
        var eventId = await ReadEventIdAsync(session, ct);
        var call = ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), method, eventId, ct);
        if (mode == "Warm" && standby)
        {
            (await Should.ThrowAsync<ServiceResultException>(call)).StatusCode.ShouldBe(StatusCodes.BadInvalidState);
            routed.ShouldBe(0, "nothing is routed to the engine");
            var acked = await TranslateAsync(session, ConditionNodeId(session, native: false), "AckedState/Id", ct);
            (await ReadAsync(session, acked, ct)).ClrValue().ShouldBe(false, "no condition state changed");
        }
        else
        {
            await call;
            routed.ShouldBe(1);
        }
    }

    private static async Task<byte[]> ReadEventIdAsync(ISession session, CancellationToken ct, bool native = false)
    {
        var eventId = await TranslateAsync(session, ConditionNodeId(session, native), BrowseNames.EventId, ct);
        return ((ByteString)(await ReadAsync(session, eventId, ct)).WrappedValue.AsBoxedObject()!).ToArray()!;
    }

    /// <summary>Owner answer Q4 for a NATIVE condition: refused on the Warm Standby ahead of the native-ack Primary gate,
    /// nothing routed to the driver; accepted once Active (positive control).</summary>
    [Theory]
    [InlineData(true)]
    [InlineData(false)]
    public async Task A_native_Acknowledge_on_a_Warm_standby_is_refused_and_never_routed(bool standby)
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync($"Warm717NativeAck{standby}", "Warm", ct);
        rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
        rig.MaterialiseNative();
        var routed = 0;
        rig.Nm.NativeAlarmAckRouter = _ => Interlocked.Increment(ref routed);
        var projection = rig.Server.CreateStandbyProjection()!;
        Promote(projection);
        rig.Project(native: true, ConditionMethodEventTests.PreState("Acknowledge", native: true) with { Message = "base" });
        if (standby) Demote(projection);

        using var session = await rig.ConnectAsync(anonymous: false, ct);
        var eventId = await ReadEventIdAsync(session, ct, native: true);
        var call = ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: true), "Acknowledge", eventId, ct);
        if (standby)
        {
            (await Should.ThrowAsync<ServiceResultException>(call)).StatusCode.ShouldBe(StatusCodes.BadInvalidState);
            routed.ShouldBe(0);
        }
        else
        {
            await call;
            routed.ShouldBe(1);
        }
    }

    internal static async Task<NodeId> TranslateAsync(ISession session, NodeId start, string path, CancellationToken ct)
    {
        var elements = path.Split('/').Select(name => new RelativePathElement
        {
            ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, IncludeSubtypes = true, TargetName = new QualifiedName(name),
        }).ToArray();
        var r = await ((SessionClient)session).TranslateBrowsePathsToNodeIdsAsync(H(), new[]
        {
            new BrowsePath { StartingNode = start, RelativePath = new RelativePath { Elements = elements } },
        }, ct);
        r.Results[0].StatusCode.ShouldBe(StatusCodes.Good, path);
        return ExpandedNodeId.ToNodeId(r.Results[0].Targets[0].TargetId, session.NamespaceUris);
    }

    // ------------------------------------------------------------------ T9 alarm suppression + refresh

    [Fact]
    public async Task Standby_suppresses_new_alarm_events_on_every_item_kind_and_promotion_refreshes_them_all()
    {
        var ct = Deadline(120);
        await using var rig = await StartModeAsync("Warm717Alarms", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
        rig.MaterialiseNative();
        rig.MaterialiseScripted();

        using var session = await rig.ConnectAsync(anonymous: false, ct);
        var events = new RecordedEvents();
        var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
        var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
        var folderItem = RecordedEvents.AddEventItem(subscription, UnsNodeId(session, EqA));
        var deviceItem = RecordedEvents.AddEventItem(subscription,
            new NodeId(RawDeviceFolder, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri)));
        await subscription.ApplyChangesAsync(ct);

        // While Standby (boot state): a new transition reaches no item, but the condition node is current.
        rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "while-standby"));
        // A condition raised and cleared (Retain=false) while Standby is not replayed by the promotion refresh.
        rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash"));
        rig.Project(native: false, new AlarmConditionSnapshot(Active: false, Acknowledged: true, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "flash-cleared"));
        await Task.Delay(1000, ct);
        foreach (var item in new[] { serverItem, folderItem, deviceItem })
        {
            events.Has(item, "while-standby").ShouldBeFalse($"no alarm event on {item.StartNodeId} while Standby");
            events.Has(item, "flash").ShouldBeFalse();
        }

        // Model-change events are not alarm events: still delivered on the Standby.
        rig.Sink.RaiseNodesAddedModelChange(EqA, AddressSpaceRealm.Uns);
        await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.GeneralModelChangeEventType));

        // D5: ConditionRefresh2 on one item, then ConditionRefresh on the subscription, on the Standby: Start, the
        // retained condition, End.
        await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct,
            subscription.Id, folderItem.Status.Id);
        await RecordedEvents.WaitUntilAsync(() => events.HasType(folderItem, ObjectTypeIds.RefreshEndEventType));
        events.Has(folderItem, "while-standby").ShouldBeTrue("ConditionRefresh2 on the Standby returns the retained condition");
        // D5: a client ConditionRefresh on the Standby returns Start, the retained condition, End.
        await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
        await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType));
        events.Has(serverItem, "while-standby").ShouldBeTrue("the Standby's refresh carries the retained condition");
        events.Has(serverItem, "flash-cleared").ShouldBeFalse("a non-retained condition is not replayed");
        foreach (var item in new[] { folderItem, deviceItem })
            await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
        var countsBefore = new[] { serverItem, folderItem, deviceItem }.ToDictionary(i => i, i => events.Of(i).Count);

        Promote(projection);
        foreach (var item in new[] { serverItem, folderItem, deviceItem })
        {
            await RecordedEvents.WaitUntilAsync(() => events.Of(item).Skip(countsBefore[item])
                .Any(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType));
            var stream = events.Of(item).Skip(countsBefore[item]).ToList();
            var start = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
            var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
            start.ShouldBeGreaterThanOrEqualTo(0, $"RefreshStart on {item.StartNodeId}");
            var replayed = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList();
            replayed.ShouldContain("while-standby", $"promotion refresh replays the retained condition on {item.StartNodeId}");
            replayed.ShouldNotContain("flash-cleared");
        }

        // After promotion new events flow, and the Standby-stamped EventId acknowledges.
        rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 800, Message: "after"));
        await RecordedEvents.WaitUntilAsync(() => events.Has(serverItem, "after") && events.Has(folderItem, "after") && events.Has(deviceItem, "after"));
        await subscription.DeleteAsync(true, ct);
    }

    [Fact]
    public async Task A_Standby_stamped_EventId_is_accepted_by_Acknowledge_after_promotion()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717AckAfter", "Warm", ct);
        rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
        rig.MaterialiseScripted();
        var routed = new List<AlarmCommand>();
        rig.Nm.AlarmCommandRouter = cmd => { lock (routed) routed.Add(cmd); };
        rig.Project(native: false, ConditionMethodEventTests.PreState("Acknowledge", native: false) with { Message = "stamped" });

        using var session = await rig.ConnectAsync(anonymous: false, ct);
        var stamped = await ReadEventIdAsync(session, ct); // stamped while Standby, never delivered
        Promote(rig.Server.CreateStandbyProjection()!);
        await ConditionMethodEventTests.CallAsync(session, ConditionNodeId(session, native: false), "Acknowledge", stamped, ct);
        lock (routed) routed.ShouldHaveSingleItem().Operation.ShouldBe("Acknowledge");
    }

    /// <summary>Spec §4.5 boundary: an event REPORTED while Active but still deferred (#838) at demotion is delivered.</summary>
    [Fact]
    public async Task An_event_reported_while_Active_but_deferred_is_still_delivered_after_demotion()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717Deferred", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
        rig.MaterialiseNative();
        Promote(projection);

        using var session = await rig.ConnectAsync(anonymous: false, ct);
        var events = new RecordedEvents();
        var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
        var onA = RecordedEvents.AddEventItem(subscription, UnsNodeId(session, EqA));
        await subscription.ApplyChangesAsync(ct);
        rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "seed"));
        await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed"));

        using (var stall = NotifierStall.Hold(rig.Nm, EqA))
        {
            rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
                Shelving: AlarmShelvingKind.Unshelved, Severity: 710, Message: "reported-active"));
            rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the report must actually be deferred");
            Demote(projection);
            rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
                Shelving: AlarmShelvingKind.Unshelved, Severity: 720, Message: "reported-standby"));
            stall.Release();
        }

        await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "reported-active"));
        await Task.Delay(500, ct);
        events.Has(onA, "reported-standby").ShouldBeFalse("a report made on the Standby is suppressed");
    }

    /// <summary>The promotion refresh goes through the per-item ACL decision: a condition the session may not see is
    /// not replayed, while one it may see is (positive control).</summary>
    [Fact]
    public async Task The_promotion_refresh_withholds_an_ACL_denied_condition()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717AclRefresh", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        rig.MaterialiseNative();
        rig.MaterialiseScripted();
        rig.Enforce(true, [Key()], EquipmentGrant(EqB,
            NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead));
        rig.Project(native: true, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "native-visible"));
        rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "scripted-denied"));

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        var events = new RecordedEvents();
        var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
        var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
        await subscription.ApplyChangesAsync(ct);

        Promote(projection);
        await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType));
        events.Has(serverItem, "native-visible").ShouldBeTrue("positive control: a visible retained condition is replayed");
        events.Has(serverItem, "scripted-denied").ShouldBeFalse("an ACL-denied condition is withheld from the promotion refresh");
    }

    /// <summary>T1: the redundancy check uses the EFFECTIVE ApplicationUri an ApplicationConfigPath XML load produces, not
    /// the options value (c7787): listing the options URI as a peer is fine, listing the XML URI fails start.</summary>
    [Fact]
    public async Task The_effective_application_uri_from_a_loaded_xml_configuration_is_what_Warm_validates_and_advertises()
    {
        var ct = Deadline();
        var seed = await ServerEventRig.StartAsync("Warm717XmlSeed", encrypted: false, ct);
        var saved = seed.HostConfiguration;
        var dir = Path.Combine(Path.GetTempPath(), $"warm717-xml-{Guid.NewGuid():N}");
        Directory.CreateDirectory(dir);
        var file = Path.Combine(dir, "OtOpcUa.Config.xml");
        var xmlUri = saved.ApplicationUri;
        try
        {
            var port = OpcUaHostStart.AllocateFreePort();
            saved.ServerConfiguration!.BaseAddresses = [$"opc.tcp://127.0.0.1:{port}/OtOpcUa"];
            saved.SaveToFile(file);

            OpcUaApplicationHostOptions Options(string peer) => new()
            {
                ApplicationName = saved.ApplicationName ?? "OtOpcUa.Warm717Xml",
                ApplicationUri = "urn:OtOpcUa:options-value",
                ApplicationConfigPath = file,
                PublicHostname = "127.0.0.1",
                OpcUaPort = port,
                PkiStoreRoot = dir,
                AutoAcceptUntrustedClientCertificates = true,
                PeerApplicationUris = [peer],
                Redundancy = new OpcUaRedundancyOptions { Mode = "Warm" },
            };

            var refused = new OpcUaApplicationHost(Options(xmlUri!),
                Microsoft.Extensions.Logging.Abstractions.NullLogger<OpcUaApplicationHost>.Instance);
            (await Should.ThrowAsync<InvalidOperationException>(refused.StartAsync(new OtOpcUaSdkServer(), ct)))
                .Message.ShouldContain(xmlUri!);

            await using var host = new OpcUaApplicationHost(Options("urn:OtOpcUa:options-value"),
                Microsoft.Extensions.Logging.Abstractions.NullLogger<OpcUaApplicationHost>.Instance);
            var server = new OtOpcUaSdkServer();
            await host.StartAsync(server, ct);
            var redundancy = server.CurrentInstance!.ServerObject.ServerRedundancy.ShouldBeOfType<NonTransparentRedundancyState>();
            var uris = redundancy.ServerUriArray!.Value.ToArray()!;
            uris.ShouldBe(new[] { xmlUri!, "urn:OtOpcUa:options-value" }, "self is the effective (XML) URI");
        }
        finally
        {
            // The seed stays up until here: the saved XML points at its PKI store (and certificate thumbprint).
            await seed.DisposeAsync();
            try { Directory.Delete(dir, recursive: true); } catch (IOException) { }
        }
    }

    // ------------------------------------------------------------------ T8 live items on transition

    [Fact]
    public async Task Demotion_queues_one_BadOutOfService_then_silence_and_promotion_queues_the_current_value_through_a_deadband()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717Live", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        var raw = rig.EnsureRawVariable("live", "Double", writable: false);
        rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        Promote(projection);

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        var client = (SessionClient)session;
        var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
        await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
        await AddItemAsync(client, sub, Raw(session, raw), 2, 10, ct, new DataChangeFilter
        {
            Trigger = DataChangeTrigger.StatusValue, DeadbandType = (uint)DeadbandType.Absolute, DeadbandValue = 1000,
        });
        (await CollectAsync(client, ct, TimeSpan.FromSeconds(1))).Where(x => x.Handle is 1 or 2).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);

        Demote(projection);
        rig.Sink.WriteValue(raw, 2.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        rig.Sink.WriteValue(raw, 3.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        var demoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2));
        demoted.Count(x => x.Handle == 1).ShouldBe(1, "exactly one notification per item, then silence");
        demoted.Count(x => x.Handle == 2).ShouldBe(1);
        demoted.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);

        rig.Sink.WriteValue(raw, 4.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        Promote(projection);
        var promoted = await CollectAsync(client, ct, TimeSpan.FromSeconds(2));
        promoted.ShouldContain(x => x.Handle == 1 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0));
        promoted.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good && Equals(x.Value.ClrValue(), 4.0),
            "the promotion value is queued ignoring the deadband");
    }

    // ------------------------------------------------------------------ T7 first-delivery decision record (D4)

    /// <summary>Timing 1 (T0): Good samples queued in the item while Active, a client-handle change, then demotion.</summary>
    [Fact]
    public async Task Timing1_queued_good_samples_then_demotion_first_delivery_is_projected_and_republish_matches()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717T1", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        var raw = rig.EnsureRawVariable("q", "Double", writable: false);
        rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        rig.MaterialiseScripted();
        Promote(projection);

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        var client = (SessionClient)session;
        var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
        var tagItem = await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
        await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, 2, 1, ct);
        var severity = await TranslateAsync(session, ConditionNodeId(session, native: false), BrowseNames.Severity, ct);
        await AddItemAsync(client, sub, severity, 3, 1, ct);
        (await PublishDataAsync(client, ct)).NotificationMessage.NotificationData.Count.ShouldBeGreaterThan(0);

        await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
        rig.Sink.WriteValue(raw, 10.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        rig.Sink.WriteValue(raw, 11.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        // Non-ordinary changes queued in the same window: i=2267 and a condition child (Severity).
        new SdkServiceLevelPublisher(rig.Server.CurrentInstance!,
            Microsoft.Extensions.Logging.Abstractions.NullLogger<SdkServiceLevelPublisher>.Instance).Publish(90);
        rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 650, Message: "child"));
        await client.ModifyMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[]
        {
            new MonitoredItemModifyRequest { MonitoredItemId = tagItem,
                RequestedParameters = new MonitoringParameters { ClientHandle = 99, SamplingInterval = 0, QueueSize = 10, DiscardOldest = true } },
        }, ct);
        DemoteDeliveryOnly(projection);
        await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);
        var msg = await PublishDataAsync(client, ct);
        var changes = DataChanges(msg.NotificationMessage);
        changes.Where(x => x.Handle is 1 or 99).ShouldNotBeEmpty();
        changes.Where(x => x.Handle is 1 or 99).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService && x.Value.WrappedValue.IsNull,
            "Good samples queued before demotion are first-delivered projected (D4)");
        changes.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good, "i=2267 is not an ordinary tag");
        changes.ShouldContain(x => x.Handle == 3 && x.Value.StatusCode == StatusCodes.Good, "a condition child is not an ordinary tag");

        var again = await client.RepublishAsync(H(), sub, msg.NotificationMessage.SequenceNumber, ct);
        DataChanges(again.NotificationMessage).Where(x => x.Handle is 1 or 99)
            .ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService, "Republish returns the recorded projected choice");
    }

    /// <summary>Timing 2 (T0): messages prebuilt while Active (one notification per message), then demotion.</summary>
    [Fact]
    public async Task Timing2_prebuilt_messages_then_demotion_next_publish_is_projected_and_unsent_republish_is_unavailable()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717T2", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        var raw = rig.EnsureRawVariable("p", "Double", writable: false);
        rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        Promote(projection);

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        var client = (SessionClient)session;
        var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 1, true, 0, ct)).SubscriptionId;
        await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
        DataChanges((await PublishDataAsync(client, ct)).NotificationMessage).Count.ShouldBe(1);

        await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
        rig.Sink.WriteValue(raw, 20.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        rig.Sink.WriteValue(raw, 21.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        rig.Sink.WriteValue(raw, 22.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);
        var a = await PublishDataAsync(client, ct); // builds several messages while Active; the first is delivered Good
        DataChanges(a.NotificationMessage).Single().Value.StatusCode.ShouldBe(StatusCodes.Good);

        DemoteDeliveryOnly(projection);
        var next = a.NotificationMessage.SequenceNumber + 1;
        (await Should.ThrowAsync<ServiceResultException>(client.RepublishAsync(H(), sub, next, ct).AsTask())).StatusCode
            .ShouldBe(StatusCodes.BadMessageNotAvailable, "2.0.1: a prebuilt, never-sent message cannot be first-delivered by Republish");
        var b = await PublishDataAsync(client, ct);
        b.NotificationMessage.SequenceNumber.ShouldBe(next);
        DataChanges(b.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService,
            "a message built while Active but first delivered after demotion is projected");
        var bRe = await client.RepublishAsync(H(), sub, next, ct);
        DataChanges(bRe.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);
    }

    /// <summary>Timing 3 (T0): Good delivered while Active, then demotion — Republish returns the original Good; after an
    /// acknowledge it is gone.</summary>
    [Fact]
    public async Task Timing3_good_delivered_while_active_republishes_good_after_demotion_and_ack_drops_it()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717T3", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        var raw = rig.EnsureRawVariable("d", "Double", writable: false);
        rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        Promote(projection);

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        var client = (SessionClient)session;
        var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
        await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
        var first = await PublishDataAsync(client, ct);
        DataChanges(first.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);

        Demote(projection);
        var again = await client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct);
        DataChanges(again.NotificationMessage).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good,
            "Republish of a message first delivered while Active returns the original (D4 boundary)");

        await client.PublishAsync(H(), new[] { new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = first.NotificationMessage.SequenceNumber } }, ct);
        (await Should.ThrowAsync<ServiceResultException>(client.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct).AsTask()))
            .StatusCode.ShouldBe(StatusCodes.BadMessageNotAvailable);
    }

    /// <summary>The D4 boundary alone (spec §4.7 demotion step 2), without the projection that follows it, so the
    /// decision record is what the test observes.</summary>
    internal static void DemoteDeliveryOnly(IStandbyProjection p) => p.SetStandbyDelivery(true);

    // ------------------------------------------------------------------ raw client helpers

    internal static async Task<uint> AddItemAsync(SessionClient client, uint sub, NodeId node, uint handle, uint queue, CancellationToken ct,
        MonitoringFilter? filter = null)
    {
        var r = await client.CreateMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[]
        {
            new MonitoredItemCreateRequest
            {
                ItemToMonitor = new ReadValueId { NodeId = node, AttributeId = Attributes.Value },
                MonitoringMode = MonitoringMode.Reporting,
                RequestedParameters = new MonitoringParameters
                {
                    ClientHandle = handle, SamplingInterval = 0, QueueSize = queue, DiscardOldest = true,
                    Filter = filter is null ? ExtensionObject.Null : new ExtensionObject(filter),
                },
            },
        }, ct);
        StatusCode.IsGood(r.Results[0].StatusCode).ShouldBeTrue($"create {node}: {r.Results[0].StatusCode}");
        return r.Results[0].MonitoredItemId;
    }

    internal static List<(uint Handle, DataValue Value)> DataChanges(NotificationMessage m)
    {
        var list = new List<(uint, DataValue)>();
        for (var i = 0; i < m.NotificationData.Count; i++)
        {
            if (m.NotificationData[i].TryGetValue<DataChangeNotification>(out var d))
                for (var j = 0; j < d.MonitoredItems.Count; j++) list.Add((d.MonitoredItems[j].ClientHandle, d.MonitoredItems[j].Value));
        }

        return list;
    }

    internal static async Task<PublishResponse> PublishDataAsync(SessionClient client, CancellationToken ct)
    {
        for (var i = 0; i < 40; i++)
        {
            var r = await client.PublishAsync(H(), default, ct);
            if (r.NotificationMessage.NotificationData.Count > 0) return r;
        }

        throw new InvalidOperationException("no data message");
    }

    internal static async Task<List<(uint Handle, DataValue Value)>> CollectAsync(SessionClient client, CancellationToken ct, TimeSpan window)
    {
        var collected = new List<(uint, DataValue)>();
        var until = DateTime.UtcNow + window;
        while (DateTime.UtcNow < until)
        {
            var r = await client.PublishAsync(H(), default, ct);
            collected.AddRange(DataChanges(r.NotificationMessage));
        }

        return collected;
    }
}