ScadaBridge/src/ZB.MOM.WW.ScadaBridge.Host/SiteLocalDbLegacyMigrator.cs

using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Configuration;
using ZB.MOM.WW.LocalDb;

namespace ZB.MOM.WW.ScadaBridge.Host;

/// <summary>
/// One-time copy of the pre-consolidation site databases — Phase 1's
/// <c>site-tracking.db</c> and <c>site_events.db</c>, and Phase 2's
/// <c>store-and-forward.db</c> — into the consolidated <c>ZB.MOM.WW.LocalDb</c> database.
/// </summary>
/// <remarks>
/// <para>
/// <b>Runs after <c>RegisterReplicated</c>, deliberately.</b> Capture is trigger-based, so
/// rows inserted before registration would never enter the oplog and would never reach the
/// peer. Migrating after registration means the migrated history replicates like any other
/// write — which is the entire point of Phase 1.
/// </para>
/// <para>
/// <b>Idempotent twice over.</b> Tracking rows carry the same TEXT primary key they always
/// had, so <c>INSERT OR IGNORE</c> is naturally idempotent. Legacy events had autoincrement
/// integer ids, which the consolidated schema replaced with GUIDs; they are given
/// <i>deterministic</i> ids of the form <c>mig-{NodeName}-{legacyId}</c> rather than fresh
/// GUIDs, so a migration that crashes and re-runs cannot duplicate events. The NodeName
/// prefix keeps the two nodes of a pair from colliding on their independent legacy id
/// sequences.
/// </para>
/// <para>
/// <b>All-or-nothing.</b> The copy runs in one transaction and the legacy file is renamed
/// to <c>&lt;name&gt;.migrated</c> only after commit. A failure throws out of
/// <c>OnReady</c>, which fails host startup with the legacy files untouched — no
/// half-migrated state to reason about. A second boot sees the renamed file and no-ops.
/// </para>
/// <para>
/// <b>Finding nothing is the expected case on the docker rig for the two Phase 1 files.</b>
/// Neither legacy config key
/// is set in any rig appsettings, so both fall back to CWD-relative code defaults
/// (<c>/app/site-tracking.db</c>, <c>/app/site_events.db</c>) that sit OUTSIDE the mounted
/// data volume — meaning they were already being discarded on every container recreate.
/// Phase 1 incidentally fixes that data-loss bug by consolidating into
/// <c>/app/data/site-localdb.db</c>. A no-op here is a legitimate result, not a failure.
/// </para>
/// <para>
/// <b>Store-and-forward is the exception.</b> Its default path <i>is</i> inside the data
/// volume (<c>./data/store-and-forward.db</c>), so a real deployment has a real file there
/// holding undelivered messages. That migration genuinely moves data, and dropping it would
/// silently discard exactly the buffered calls store-and-forward exists to protect.
/// </para>
/// </remarks>
public static class SiteLocalDbLegacyMigrator
{
    private const string MigratedSuffix = ".migrated";

    /// <summary>Default legacy tracking connection string (<c>OperationTrackingOptions.ConnectionString</c>).</summary>
    private const string DefaultTrackingConnectionString = "Data Source=site-tracking.db";

    /// <summary>Default legacy event-log path (<c>SiteEventLogOptions.DatabasePath</c>).</summary>
    private const string DefaultEventLogPath = "site_events.db";

    /// <summary>Default legacy store-and-forward path (<c>StoreAndForwardOptions.SqliteDbPath</c>).</summary>
    private const string DefaultStoreAndForwardPath = "./data/store-and-forward.db";

    /// <summary>Default legacy site configuration path (<c>appsettings.Site.json</c>).</summary>
    private const string DefaultSiteStoragePath = "./data/scadabridge.db";

    /// <summary>
    /// Every column of the current <c>sf_messages</c> schema, in a fixed order. Columns
    /// absent from an older legacy file are dropped from the copy rather than failing it —
    /// see <see cref="PresentColumns"/>.
    /// </summary>
    private static readonly string[] StoreAndForwardColumns =
    [
        "id", "category", "target", "payload_json",
        "retry_count", "max_retries", "retry_interval_ms",
        "created_at", "last_attempt_at", "status", "last_error", "origin_instance",
        "execution_id", "source_script", "parent_execution_id", "last_attempt_at_ms",
    ];

    /// <summary>
    /// The site configuration tables copied out of the legacy <c>scadabridge.db</c>, with
    /// the current schema's columns for each.
    /// </summary>
    /// <remarks>
    /// <b><c>notification_lists</c> and <c>smtp_configurations</c> are deliberately absent.</b>
    /// Both are purged on every deploy and are permanently empty by design — the site-side
    /// write paths were removed on 2026-07-10. A pre-fix legacy file can still hold rows, and
    /// <c>smtp_configurations.password</c> is plaintext.
    /// <para>
    /// Skipping them here is one half of a pair: the cutover also declines to register them
    /// for replication, for the same reason. Migrating them would leave plaintext SMTP
    /// passwords sitting in the consolidated database — one future <c>RegisterReplicated</c>
    /// away from being shipped to a peer — in exchange for resurrecting config that nothing
    /// reads. Keeping the tables permanently empty is what makes both decisions safe.
    /// </para>
    /// The tables themselves are still created (see <c>SiteStorageSchema</c>); only their
    /// historical contents are left behind.
    /// </remarks>
    internal static readonly LegacyTable[] SiteStorageTables =
    [
        new("deployed_configurations", "instance_unique_name",
        [
            "instance_unique_name", "config_json", "deployment_id", "revision_hash",
            "is_enabled", "deployed_at",
        ]),
        new("static_attribute_overrides", "instance_unique_name",
        [
            "instance_unique_name", "attribute_name", "override_value", "updated_at",
        ]),
        new("shared_scripts", "name",
        [
            "name", "code", "parameter_definitions", "return_definition", "updated_at",
        ]),
        new("external_systems", "name",
        [
            "name", "endpoint_url", "auth_type", "auth_configuration", "method_definitions",
            "updated_at", "timeout_seconds",
        ]),
        new("database_connections", "name",
        [
            "name", "connection_string", "max_retries", "retry_delay_ms", "updated_at",
        ]),
        new("data_connection_definitions", "name",
        [
            "name", "protocol", "configuration", "backup_configuration",
            "failover_retry_count", "updated_at",
        ]),
        new("native_alarm_state", "instance_unique_name",
        [
            "instance_unique_name", "source_canonical_name", "source_reference",
            "condition_json", "last_transition_at", "metadata_json",
        ]),
    ];

    /// <summary>
    /// Copies any legacy site databases into <paramref name="db"/>, then renames them.
    /// </summary>
    /// <param name="db">The consolidated site database, with both tables already registered.</param>
    /// <param name="config">Configuration supplying the legacy paths and the node name.</param>
    public static void Migrate(ILocalDb db, IConfiguration config)
    {
        ArgumentNullException.ThrowIfNull(db);
        ArgumentNullException.ThrowIfNull(config);

        var nodeName = config["ScadaBridge:Node:NodeName"] ?? "unknown-node";

        MigrateTracking(db, ResolveTrackingPath(config));
        MigrateEvents(db, ResolveEventLogPath(config), nodeName);
        MigrateStoreAndForward(db, ResolveStoreAndForwardPath(config));
        MigrateSiteStorage(db, ResolveSiteStoragePath(config));
    }

    /// <summary>
    /// Resolves the legacy tracking database path from the OLD connection-string key,
    /// falling back to the code default. Relative paths resolve against the process working
    /// directory — NOT the data volume — because that is where the old code actually put them.
    /// </summary>
    internal static string ResolveTrackingPath(IConfiguration config)
    {
        var connectionString =
            config["ScadaBridge:OperationTracking:ConnectionString"] ?? DefaultTrackingConnectionString;

        string dataSource;
        try
        {
            dataSource = new SqliteConnectionStringBuilder(connectionString).DataSource;
        }
        catch (ArgumentException)
        {
            // An unparseable legacy connection string means there is nothing to migrate
            // from. Do not fail the host over a stale key we are about to stop reading.
            return string.Empty;
        }

        // In-memory legacy databases (test/dev configurations) have nothing durable to
        // migrate, and Path.GetFullPath on them would produce nonsense.
        if (string.IsNullOrWhiteSpace(dataSource) ||
            dataSource.Equals(":memory:", StringComparison.OrdinalIgnoreCase) ||
            dataSource.StartsWith("file:", StringComparison.OrdinalIgnoreCase))
        {
            return string.Empty;
        }

        return Path.GetFullPath(dataSource);
    }

    /// <summary>Resolves the legacy event-log path from the OLD key, falling back to the code default.</summary>
    internal static string ResolveEventLogPath(IConfiguration config)
    {
        var path = config["ScadaBridge:SiteEventLog:DatabasePath"] ?? DefaultEventLogPath;

        if (string.IsNullOrWhiteSpace(path) ||
            path.Equals(":memory:", StringComparison.OrdinalIgnoreCase))
        {
            return string.Empty;
        }

        return Path.GetFullPath(path);
    }

    /// <summary>
    /// Resolves the legacy store-and-forward database path from the OLD key, falling back
    /// to the code default. Unlike the two Phase 1 paths, this default sits INSIDE the
    /// mounted data volume (<c>./data/</c>), so on the docker rig there is a real file here
    /// with real buffered messages — this migration is not the usual no-op.
    /// </summary>
    internal static string ResolveStoreAndForwardPath(IConfiguration config)
    {
        var path = config["ScadaBridge:StoreAndForward:SqliteDbPath"] ?? DefaultStoreAndForwardPath;

        if (string.IsNullOrWhiteSpace(path) ||
            path.Equals(":memory:", StringComparison.OrdinalIgnoreCase) ||
            path.StartsWith("file:", StringComparison.OrdinalIgnoreCase))
        {
            return string.Empty;
        }

        return Path.GetFullPath(path);
    }

    /// <summary>
    /// Resolves the legacy site configuration database path from the OLD key, falling back
    /// to the code default. Like store-and-forward — and unlike the two Phase 1 paths — this
    /// default is inside the mounted data volume, so a real deployment has real config here.
    /// </summary>
    internal static string ResolveSiteStoragePath(IConfiguration config)
    {
        var path = config["ScadaBridge:Database:SiteDbPath"] ?? DefaultSiteStoragePath;

        if (string.IsNullOrWhiteSpace(path) ||
            path.Equals(":memory:", StringComparison.OrdinalIgnoreCase) ||
            path.StartsWith("file:", StringComparison.OrdinalIgnoreCase))
        {
            return string.Empty;
        }

        return Path.GetFullPath(path);
    }

    /// <summary>
    /// Copies buffered store-and-forward messages out of the legacy file.
    /// </summary>
    /// <remarks>
    /// No id synthesis, unlike <see cref="MigrateEvents"/>: <c>sf_messages.id</c> is already
    /// a caller-assigned TEXT primary key, so <c>INSERT OR IGNORE</c> is naturally idempotent
    /// across a crash-then-rerun.
    /// <para>
    /// These are undelivered messages, so dropping them is real data loss — a buffered call
    /// that never reaches its external system is exactly what store-and-forward exists to
    /// prevent. That is why the copy tolerates an older column set rather than bailing.
    /// </para>
    /// </remarks>
    private static void MigrateStoreAndForward(ILocalDb db, string legacyPath)
        => MigrateFile(db, legacyPath, [new LegacyTable("sf_messages", "id", StoreAndForwardColumns)]);

    /// <summary>
    /// Copies the site's configuration tables out of the legacy <c>scadabridge.db</c>.
    /// </summary>
    /// <remarks>
    /// All seven migrated tables live in one file, so they are copied inside a single
    /// transaction and the file is renamed once: a partial config migration would leave a
    /// site node running against half its old configuration, which is worse than failing
    /// startup outright.
    /// </remarks>
    private static void MigrateSiteStorage(ILocalDb db, string legacyPath)
        => MigrateFile(db, legacyPath, SiteStorageTables);

    /// <summary>
    /// Copies one table from a legacy file into the consolidated database, then renames the
    /// legacy file.
    /// </summary>
    /// <remarks>
    /// The copy is restricted to the columns the legacy table <i>actually has</i>. A file
    /// written by an older build predates some columns, and naming a missing column in the
    /// SELECT would throw "no such column" — which the reader treats as an unrecognised
    /// shape, silently discarding every row in the table. Intersecting first means an old
    /// file migrates its data and simply leaves the newer columns at their schema defaults.
    /// </remarks>
    /// <param name="db">The consolidated database.</param>
    /// <param name="legacyPath">The legacy file, which may not exist.</param>
    /// <param name="tables">Every table to copy out of this file, in order.</param>
    private static void MigrateFile(ILocalDb db, string legacyPath, IReadOnlyList<LegacyTable> tables)
    {
        if (!ShouldMigrate(legacyPath)) return;

        using (var legacy = OpenLegacyReadOnly(legacyPath))
        {
            using var connection = db.CreateConnection();
            using var transaction = connection.BeginTransaction();

            foreach (var table in tables)
            {
                var present = PresentColumns(legacy, table.Table, table.Columns);

                // An absent table probes as zero columns, so this one guard covers both
                // "old file predating the table" and "file we do not recognise".
                if (present.Contains(table.RequiredColumn))
                    CopyRows(legacy, connection, transaction, table.Table, present);
            }

            transaction.Commit();
        }

        MarkMigrated(legacyPath);
    }

    /// <summary>One table to copy out of a legacy file.</summary>
    /// <param name="Table">Table name, identical on both sides.</param>
    /// <param name="RequiredColumn">
    /// A column without which the table is not the one we mean — normally the primary key.
    /// Copying rows with a NULL PK would be worse than copying nothing.
    /// </param>
    /// <param name="Columns">The current schema's full column list, in a fixed order.</param>
    internal sealed record LegacyTable(string Table, string RequiredColumn, string[] Columns);

    private static void MigrateTracking(ILocalDb db, string legacyPath)
    {
        if (!ShouldMigrate(legacyPath)) return;

        var rows = ReadAll(
            legacyPath,
            """
            SELECT TrackedOperationId, Kind, TargetSummary, Status,
                   RetryCount, LastError, HttpStatus,
                   CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc,
                   SourceInstanceId, SourceScript, SourceNode
            FROM   OperationTracking;
            """,
            expectedColumns: 13);

        // A legacy file with no OperationTracking table at all reads as "nothing to do";
        // it still gets renamed so the probe does not repeat on every boot.
        if (rows is not null)
        {
            using var connection = db.CreateConnection();
            using var transaction = connection.BeginTransaction();

            foreach (var row in rows)
            {
                using var cmd = connection.CreateCommand();
                cmd.Transaction = transaction;
                cmd.CommandText = """
                    INSERT OR IGNORE INTO OperationTracking (
                        TrackedOperationId, Kind, TargetSummary, Status,
                        RetryCount, LastError, HttpStatus,
                        CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc,
                        SourceInstanceId, SourceScript, SourceNode
                    ) VALUES (
                        $p0, $p1, $p2, $p3, $p4, $p5, $p6,
                        $p7, $p8, $p9, $p10, $p11, $p12
                    );
                    """;
                Bind(cmd, row);
                cmd.ExecuteNonQuery();
            }

            transaction.Commit();
        }

        MarkMigrated(legacyPath);
    }

    private static void MigrateEvents(ILocalDb db, string legacyPath, string nodeName)
    {
        if (!ShouldMigrate(legacyPath)) return;

        var rows = ReadAll(
            legacyPath,
            """
            SELECT id, timestamp, event_type, severity, instance_id, source, message, details
            FROM   site_events;
            """,
            expectedColumns: 8);

        if (rows is not null)
        {
            using var connection = db.CreateConnection();
            using var transaction = connection.BeginTransaction();

            foreach (var row in rows)
            {
                using var cmd = connection.CreateCommand();
                cmd.Transaction = transaction;
                cmd.CommandText = """
                    INSERT OR IGNORE INTO site_events (
                        id, timestamp, event_type, severity, instance_id, source, message, details
                    ) VALUES (
                        $id, $p1, $p2, $p3, $p4, $p5, $p6, $p7
                    );
                    """;

                // Deterministic id, NOT a fresh GUID: a crashed-then-rerun migration must
                // not duplicate events, and INSERT OR IGNORE can only dedupe on a stable key.
                cmd.Parameters.AddWithValue("$id", $"mig-{nodeName}-{row[0] ?? "null"}");
                for (var i = 1; i < row.Length; i++)
                    cmd.Parameters.AddWithValue($"$p{i}", row[i] ?? (object)DBNull.Value);

                cmd.ExecuteNonQuery();
            }

            transaction.Commit();
        }

        MarkMigrated(legacyPath);
    }

    /// <summary>True when there is a legacy file present that has not already been migrated.</summary>
    private static bool ShouldMigrate(string legacyPath)
        => !string.IsNullOrEmpty(legacyPath)
           && File.Exists(legacyPath)
           && !File.Exists(legacyPath + MigratedSuffix);

    /// <summary>
    /// Reads every row of <paramref name="sql"/> from the legacy file, or null when the
    /// table does not exist (an old file predating the table is not an error).
    /// </summary>
    private static List<object?[]>? ReadAll(string legacyPath, string sql, int expectedColumns)
    {
        using var connection = new SqliteConnection($"Data Source={legacyPath};Mode=ReadOnly");
        connection.Open();

        using var cmd = connection.CreateCommand();
        cmd.CommandText = sql;

        SqliteDataReader reader;
        try
        {
            reader = cmd.ExecuteReader();
        }
        catch (SqliteException)
        {
            // "no such table" / "no such column" — a legacy file from before this table
            // existed, or a shape we do not recognise. Nothing to copy.
            return null;
        }

        var rows = new List<object?[]>();
        using (reader)
        {
            while (reader.Read())
            {
                var row = new object?[expectedColumns];
                for (var i = 0; i < expectedColumns; i++)
                    row[i] = reader.IsDBNull(i) ? null : reader.GetValue(i);
                rows.Add(row);
            }
        }

        return rows;
    }

    private static SqliteConnection OpenLegacyReadOnly(string legacyPath)
    {
        var connection = new SqliteConnection($"Data Source={legacyPath};Mode=ReadOnly");
        connection.Open();
        return connection;
    }

    /// <summary>
    /// Returns the subset of <paramref name="wanted"/> that the legacy table actually has,
    /// in the caller's order. An absent table yields an empty list rather than throwing.
    /// </summary>
    private static List<string> PresentColumns(
        SqliteConnection legacy, string table, IReadOnlyList<string> wanted)
    {
        var present = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

        using (var probe = legacy.CreateCommand())
        {
            // Table name is a caller-controlled constant, never user input — safe to
            // interpolate (parameters are not permitted as a pragma-function argument).
            probe.CommandText = $"SELECT name FROM pragma_table_info('{table}')";
            using var reader = probe.ExecuteReader();
            while (reader.Read()) present.Add(reader.GetString(0));
        }

        return [.. wanted.Where(present.Contains)];
    }

    /// <summary>Streams every row of <paramref name="columns"/> from the legacy table into the target.</summary>
    private static void CopyRows(
        SqliteConnection legacy,
        SqliteConnection target,
        SqliteTransaction transaction,
        string table,
        IReadOnlyList<string> columns)
    {
        var columnList = string.Join(", ", columns);
        var parameterList = string.Join(", ", columns.Select((_, i) => $"$p{i}"));

        using var read = legacy.CreateCommand();
        read.CommandText = $"SELECT {columnList} FROM {table};";
        using var reader = read.ExecuteReader();

        while (reader.Read())
        {
            using var write = target.CreateCommand();
            write.Transaction = transaction;
            write.CommandText =
                $"INSERT OR IGNORE INTO {table} ({columnList}) VALUES ({parameterList});";

            for (var i = 0; i < columns.Count; i++)
                write.Parameters.AddWithValue($"$p{i}", reader.IsDBNull(i) ? DBNull.Value : reader.GetValue(i));

            write.ExecuteNonQuery();
        }
    }

    private static void Bind(SqliteCommand cmd, object?[] row)
    {
        for (var i = 0; i < row.Length; i++)
            cmd.Parameters.AddWithValue($"$p{i}", row[i] ?? (object)DBNull.Value);
    }

    /// <summary>
    /// Renames the legacy file so a later boot skips it. Done only after the copy has
    /// committed; the file is kept rather than deleted so an operator can still inspect it.
    /// </summary>
    private static void MarkMigrated(string legacyPath)
        => File.Move(legacyPath, legacyPath + MigratedSuffix);
}