ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/SiteFailoverTests.cs

using System.Text.Json;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
using ZB.MOM.WW.ScadaBridge.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;

/// <summary>
/// Full-system failover testing — Site.
/// Verifies S&amp;F buffer takeover, DCL reconnection structure, alarm re-evaluation,
/// and script trigger resumption after site failover.
/// </summary>
public class SiteFailoverTests
{
    /// <summary>Verifies that buffered store-and-forward messages are retained across a restart.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Trait("Category", "Integration")]
    [Fact]
    public async Task StoreAndForward_BufferSurvivesRestart_MessagesRetained()
    {
        // Simulates site failover: messages buffered in SQLite survive process restart.
        // The standby node picks up the same SQLite file and retries pending messages.
        var dbPath = Path.Combine(Path.GetTempPath(), $"sf_failover_{Guid.NewGuid():N}.db");

        // The store takes an ILocalDb; each "node" opens its own local database over the
        // SAME file, which is how this test models the standby picking up the primary's
        // buffer after failover.
        TestLocalDb? primaryDb = null;
        TestLocalDb? standbyDb = null;

        try
        {
            // Phase 1: Buffer messages on "primary" node
            primaryDb = TestLocalDb.Create(dbPath);
            var storage = new StoreAndForwardStorage(primaryDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
            await storage.InitializeAsync();

            var message = new StoreAndForwardMessage
            {
                Id = Guid.NewGuid().ToString("N"),
                Category = StoreAndForwardCategory.ExternalSystem,
                Target = "https://api.example.com/data",
                PayloadJson = """{"temperature":42.5}""",
                RetryCount = 2,
                MaxRetries = 50,
                RetryIntervalMs = 30000,
                CreatedAt = DateTimeOffset.UtcNow,
                Status = StoreAndForwardMessageStatus.Pending,
                OriginInstanceName = "pump-station-1"
            };

            await storage.EnqueueAsync(message);

            // Phase 2: "Standby" node opens the same database (simulating failover)
            standbyDb = TestLocalDb.Create(dbPath);
            var standbyStorage = new StoreAndForwardStorage(standbyDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
            await standbyStorage.InitializeAsync();

            var pending = await standbyStorage.GetMessagesForRetryAsync();
            Assert.Single(pending);
            Assert.Equal(message.Id, pending[0].Id);
            Assert.Equal("pump-station-1", pending[0].OriginInstanceName);
            Assert.Equal(2, pending[0].RetryCount);
        }
        finally
        {
            // Dispose before deleting — the master connections anchor the WAL sidecars.
            standbyDb?.Dispose();
            primaryDb?.Dispose();
            TestLocalDb.DeleteFiles(dbPath);
        }
    }

    /// <summary>Verifies that parked store-and-forward messages survive a failover.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Trait("Category", "Integration")]
    [Fact]
    public async Task StoreAndForward_ParkedMessages_SurviveFailover()
    {
        var dbPath = Path.Combine(Path.GetTempPath(), $"sf_parked_{Guid.NewGuid():N}.db");

        TestLocalDb? primaryDb = null;
        TestLocalDb? standbyDb = null;

        try
        {
            primaryDb = TestLocalDb.Create(dbPath);
            var storage = new StoreAndForwardStorage(primaryDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
            await storage.InitializeAsync();

            var parkedMsg = new StoreAndForwardMessage
            {
                Id = Guid.NewGuid().ToString("N"),
                Category = StoreAndForwardCategory.Notification,
                Target = "alert-list",
                PayloadJson = """{"subject":"Critical alarm"}""",
                RetryCount = 50,
                MaxRetries = 50,
                RetryIntervalMs = 30000,
                CreatedAt = DateTimeOffset.UtcNow.AddHours(-1),
                LastAttemptAt = DateTimeOffset.UtcNow,
                Status = StoreAndForwardMessageStatus.Parked,
                LastError = "SMTP connection timeout",
                OriginInstanceName = "compressor-1"
            };

            await storage.EnqueueAsync(parkedMsg);

            // Standby opens same DB
            standbyDb = TestLocalDb.Create(dbPath);
            var standbyStorage = new StoreAndForwardStorage(standbyDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
            await standbyStorage.InitializeAsync();

            var (parked, count) = await standbyStorage.GetParkedMessagesAsync();
            Assert.Equal(1, count);
            Assert.Equal("SMTP connection timeout", parked[0].LastError);
        }
        finally
        {
            // Dispose before deleting — the master connections anchor the WAL sidecars.
            standbyDb?.Dispose();
            primaryDb?.Dispose();
            TestLocalDb.DeleteFiles(dbPath);
        }
    }

    /// <summary>Verifies that an incoming value triggers alarm re-evaluation and a new alarm state.</summary>
    [Fact]
    public void AlarmReEvaluation_IncomingValue_TriggersNewState()
    {
        // Structural verification: AlarmStateChanged carries all data needed for
        // re-evaluation after failover. When DCL reconnects and pushes new values,
        // the Alarm Actor evaluates from the incoming value (not stale state).
        var alarmEvent = new AlarmStateChanged(
            "pump-station-1",
            "HighPressureAlarm",
            AlarmState.Active,
            1,
            DateTimeOffset.UtcNow);

        Assert.Equal(AlarmState.Active, alarmEvent.State);
        Assert.Equal("pump-station-1", alarmEvent.InstanceUniqueName);

        // After failover, a new value triggers re-evaluation
        var clearedEvent = new AlarmStateChanged(
            "pump-station-1",
            "HighPressureAlarm",
            AlarmState.Normal,
            1,
            DateTimeOffset.UtcNow.AddSeconds(5));

        Assert.Equal(AlarmState.Normal, clearedEvent.State);
        Assert.True(clearedEvent.Timestamp > alarmEvent.Timestamp);
    }

    /// <summary>Verifies that a value change triggers its script after failover.</summary>
    [Fact]
    public void ScriptTriggerResumption_ValueChangeTriggersScript()
    {
        // Structural verification: AttributeValueChanged messages from DCL after reconnection
        // will be routed to Script Actors, which evaluate triggers based on incoming values.
        // No stale trigger state needed — triggers fire on new values.
        var valueChange = new AttributeValueChanged(
            "pump-station-1",
            "OPC:ns=2;s=Pressure",
            "Pressure",
            150.0,
            "Good",
            DateTimeOffset.UtcNow);

        Assert.Equal("Pressure", valueChange.AttributeName);
        Assert.Equal("OPC:ns=2;s=Pressure", valueChange.AttributePath);
        Assert.Equal(150.0, valueChange.Value);
        Assert.Equal("Good", valueChange.Quality);
    }

    /// <summary>Verifies that the store-and-forward buffer depth is reported after a failover.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Trait("Category", "Integration")]
    [Fact]
    public async Task StoreAndForward_BufferDepth_ReportedAfterFailover()
    {
        var dbPath = Path.Combine(Path.GetTempPath(), $"sf_depth_{Guid.NewGuid():N}.db");

        TestLocalDb? primaryDb = null;
        TestLocalDb? standbyDb = null;

        try
        {
            primaryDb = TestLocalDb.Create(dbPath);
            var storage = new StoreAndForwardStorage(primaryDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
            await storage.InitializeAsync();

            // Enqueue messages in different categories
            for (var i = 0; i < 5; i++)
            {
                await storage.EnqueueAsync(new StoreAndForwardMessage
                {
                    Id = Guid.NewGuid().ToString("N"),
                    Category = StoreAndForwardCategory.ExternalSystem,
                    Target = "api",
                    PayloadJson = "{}",
                    MaxRetries = 50,
                    RetryIntervalMs = 30000,
                    CreatedAt = DateTimeOffset.UtcNow,
                    Status = StoreAndForwardMessageStatus.Pending,
                });
            }

            for (var i = 0; i < 3; i++)
            {
                await storage.EnqueueAsync(new StoreAndForwardMessage
                {
                    Id = Guid.NewGuid().ToString("N"),
                    Category = StoreAndForwardCategory.Notification,
                    Target = "alerts",
                    PayloadJson = "{}",
                    MaxRetries = 50,
                    RetryIntervalMs = 30000,
                    CreatedAt = DateTimeOffset.UtcNow,
                    Status = StoreAndForwardMessageStatus.Pending,
                });
            }

            // After failover, standby reads buffer depths
            standbyDb = TestLocalDb.Create(dbPath);
            var standbyStorage = new StoreAndForwardStorage(standbyDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
            await standbyStorage.InitializeAsync();

            var depths = await standbyStorage.GetBufferDepthByCategoryAsync();
            Assert.Equal(5, depths[StoreAndForwardCategory.ExternalSystem]);
            Assert.Equal(3, depths[StoreAndForwardCategory.Notification]);
        }
        finally
        {
            // Dispose before deleting — the master connections anchor the WAL sidecars.
            standbyDb?.Dispose();
            primaryDb?.Dispose();
            TestLocalDb.DeleteFiles(dbPath);
        }
    }
}