ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/RecoveryDrillTests.cs

using System.Text.Json;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
using ZB.MOM.WW.ScadaBridge.TestSupport;

namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;

/// <summary>
/// Recovery drill test scaffolds.
/// Mid-deploy failover, communication drops, and site restart with persisted configs.
/// </summary>
public class RecoveryDrillTests
{
    /// <summary>Verifies that a site state query after a mid-deploy failover is followed by a successful redeploy.</summary>
    [Trait("Category", "Integration")]
    [Fact]
    public void MidDeployFailover_SiteStateQuery_ThenRedeploy()
    {
        // Scenario: Deployment in progress, central node fails over.
        // New central node queries site for current deployment state, then re-issues deploy.

        // Step 1: Deployment started
        var initialStatus = new DeploymentStatusResponse(
            "dep-1", "pump-station-1", DeploymentStatus.InProgress,
            null, DateTimeOffset.UtcNow);

        Assert.Equal(DeploymentStatus.InProgress, initialStatus.Status);

        // Step 2: Central failover — new node queries site state
        // Site reports current status (InProgress or whatever it actually is)
        var queriedStatus = new DeploymentStatusResponse(
            "dep-1", "pump-station-1", DeploymentStatus.InProgress,
            null, DateTimeOffset.UtcNow.AddSeconds(5));

        Assert.Equal(DeploymentStatus.InProgress, queriedStatus.Status);

        // Step 3: Central re-deploys with same deployment ID + revision hash
        // Idempotent: same deploymentId + revisionHash = no-op if already applied
        var redeployCommand = new DeployInstanceCommand(
            "dep-1", "pump-station-1", "abc123",
            """{"attributes":[],"scripts":[],"alarms":[]}""",
            "admin", DateTimeOffset.UtcNow.AddSeconds(10));

        Assert.Equal("dep-1", redeployCommand.DeploymentId);
        Assert.Equal("abc123", redeployCommand.RevisionHash);

        // Step 4: Site applies (idempotent — revision hash matches)
        var completedStatus = new DeploymentStatusResponse(
            "dep-1", "pump-station-1", DeploymentStatus.Success,
            null, DateTimeOffset.UtcNow.AddSeconds(15));

        Assert.Equal(DeploymentStatus.Success, completedStatus.Status);
    }

    /// <summary>Verifies that a communication drop during artifact deployment buffers the work for retry.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Trait("Category", "Integration")]
    [Fact]
    public async Task CommunicationDrop_DuringArtifactDeployment_BuffersForRetry()
    {
        // Scenario: Communication drops while deploying system-wide artifacts.
        // The deployment command is buffered by S&F and retried when connection restores.
        var dbPath = Path.Combine(Path.GetTempPath(), $"sf_commdrop_{Guid.NewGuid():N}.db");

        // The store takes an ILocalDb (LocalDb has no in-memory mode), so the buffer
        // lives in a real temp file for the duration of the drill.
        TestLocalDb? localDb = null;

        try
        {
            localDb = TestLocalDb.Create(dbPath);
            var storage = new StoreAndForwardStorage(localDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
            await storage.InitializeAsync();

            var options = new StoreAndForwardOptions
            {
                DefaultRetryInterval = TimeSpan.FromSeconds(5),
                DefaultMaxRetries = 100,
            };
            var service = new StoreAndForwardService(storage, options, NullLogger<StoreAndForwardService>.Instance);
            await service.StartAsync();

            // Register a handler that simulates communication failure
            var callCount = 0;
            service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
                _ =>
                {
                    callCount++;
                    throw new InvalidOperationException("Connection to site lost");
                });

            // Attempt delivery — should fail and buffer
            var result = await service.EnqueueAsync(
                StoreAndForwardCategory.ExternalSystem,
                "site-01/artifacts",
                """{"deploymentId":"dep-1","artifacts":["shared-script-v2"]}""");

            Assert.True(result.Accepted);
            Assert.True(result.WasBuffered);
            Assert.Equal(1, callCount);

            // Verify the message is in the buffer
            var depths = await service.GetBufferDepthAsync();
            Assert.True(depths.ContainsKey(StoreAndForwardCategory.ExternalSystem));
            Assert.Equal(1, depths[StoreAndForwardCategory.ExternalSystem]);

            await service.StopAsync();
        }
        finally
        {
            // Dispose before deleting — the master connection anchors the WAL sidecars.
            localDb?.Dispose();
            TestLocalDb.DeleteFiles(dbPath);
        }
    }

    /// <summary>Verifies that a site restart with persisted configurations rebuilds its state from SQLite.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Trait("Category", "Integration")]
    [Fact]
    public async Task SiteRestart_WithPersistedConfigs_RebuildFromSQLite()
    {
        // Scenario: Site restarts. Deployed instance configs are persisted in SQLite.
        // On startup, the Deployment Manager Actor reads configs from SQLite and
        // recreates Instance Actors.
        var dbPath = Path.Combine(Path.GetTempPath(), $"sf_restart_{Guid.NewGuid():N}.db");

        // The restarted "process" opens its own local database over the SAME file — that
        // is what this drill verifies survives.
        TestLocalDb? preRestartDb = null;
        TestLocalDb? restartedDb = null;

        try
        {
            // Pre-restart: S&F messages in buffer
            preRestartDb = TestLocalDb.Create(dbPath);
            var storage = new StoreAndForwardStorage(preRestartDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
            await storage.InitializeAsync();

            for (var i = 0; i < 3; i++)
            {
                await storage.EnqueueAsync(new StoreAndForwardMessage
                {
                    Id = $"msg-{i}",
                    Category = StoreAndForwardCategory.ExternalSystem,
                    Target = "api-endpoint",
                    PayloadJson = $$"""{"instanceName":"machine-{{i}}","value":42}""",
                    MaxRetries = 50,
                    RetryIntervalMs = 30000,
                    CreatedAt = DateTimeOffset.UtcNow,
                    Status = StoreAndForwardMessageStatus.Pending,
                    OriginInstanceName = $"machine-{i}"
                });
            }

            // Post-restart: new storage instance reads same DB
            restartedDb = TestLocalDb.Create(dbPath);
            var restartedStorage = new StoreAndForwardStorage(restartedDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
            await restartedStorage.InitializeAsync();

            var pending = await restartedStorage.GetMessagesForRetryAsync();
            Assert.Equal(3, pending.Count);

            // Verify each message retains its origin instance
            Assert.Contains(pending, m => m.OriginInstanceName == "machine-0");
            Assert.Contains(pending, m => m.OriginInstanceName == "machine-1");
            Assert.Contains(pending, m => m.OriginInstanceName == "machine-2");
        }
        finally
        {
            // Dispose before deleting — the master connections anchor the WAL sidecars.
            restartedDb?.Dispose();
            preRestartDb?.Dispose();
            TestLocalDb.DeleteFiles(dbPath);
        }
    }

    /// <summary>Verifies that redeploying the same revision hash is a no-op.</summary>
    [Fact]
    public void DeploymentIdempotency_SameRevisionHash_NoOp()
    {
        // Verify the deployment model supports idempotency via revision hash.
        // Two deploy commands with the same deploymentId + revisionHash should
        // produce the same result (site can detect the duplicate and skip).
        var cmd1 = new DeployInstanceCommand(
            "dep-1", "pump-1", "rev-abc123",
            """{"attributes":[]}""", "admin", DateTimeOffset.UtcNow);

        var cmd2 = new DeployInstanceCommand(
            "dep-1", "pump-1", "rev-abc123",
            """{"attributes":[]}""", "admin", DateTimeOffset.UtcNow.AddSeconds(30));

        Assert.Equal(cmd1.DeploymentId, cmd2.DeploymentId);
        Assert.Equal(cmd1.RevisionHash, cmd2.RevisionHash);
        Assert.Equal(cmd1.InstanceUniqueName, cmd2.InstanceUniqueName);
    }

    /// <summary>Verifies that the flattened configuration snapshot contains the revision hash.</summary>
    [Fact]
    public void FlattenedConfigSnapshot_ContainsRevisionHash()
    {
        // The FlattenedConfigurationSnapshot includes a revision hash for staleness detection.
        var snapshot = new FlattenedConfigurationSnapshot(
            "inst-1", "rev-abc123",
            """{"attributes":[],"scripts":[],"alarms":[]}""",
            DateTimeOffset.UtcNow);

        Assert.Equal("rev-abc123", snapshot.RevisionHash);
    }
}