feat(configmanager): add pipeline element management CLI commands

Add commands to manage pipeline elements (pre-scripts, transforms, post-scripts, source, destination) matching the UI functionality: - prescript/postscript: add, remove, edit, move-up, move-down - transform: add, remove, edit, move-up, move-down with type shortcuts - source: edit (connection, query, mass-query) - destination: edit (table, match columns, exclude from update) Features include 1-based indices, --dry-run support, file input for scripts/queries, and numbered element display in pipeline show output.
2026-01-28 16:08:28 -05:00
parent 6f3e12b3b4
commit 45f4ecab7d
5 changed files with 3221 additions and 10 deletions
@@ -283,24 +283,42 @@ public static class PipelineCommands
                        logger.LogInformation("  Match Columns: {Value}", string.Join(", ", pipeline.Destination.MatchColumns));
                }

-                if (verbose)
+                // Always show element counts and details when non-empty
+                if (pipeline.PreScripts.Count > 0)
                {
-                    if (pipeline.PreScripts.Count > 0)
+                    logger.LogInformation("");
+                    logger.LogInformation("Pre-Scripts ({Count}):", pipeline.PreScripts.Count);
+                    for (int i = 0; i < pipeline.PreScripts.Count; i++)
                    {
-                        logger.LogDebug("");
-                        logger.LogDebug("Pre-Scripts: {Count}", pipeline.PreScripts.Count);
+                        var script = pipeline.PreScripts[i];
+                        var scriptPreview = script.Script.Length > 50 ? script.Script[..50] + "..." : script.Script;
+                        scriptPreview = scriptPreview.Replace("\n", " ").Replace("\r", "");
+                        logger.LogInformation("  {Index}. [{Connection}] {Script}", i + 1, script.Connection, scriptPreview);
                    }
+                }

-                    if (pipeline.Transforms.Count > 0)
+                if (pipeline.Transforms.Count > 0)
+                {
+                    logger.LogInformation("");
+                    logger.LogInformation("Transforms ({Count}):", pipeline.Transforms.Count);
+                    for (int i = 0; i < pipeline.Transforms.Count; i++)
                    {
-                        logger.LogDebug("");
-                        logger.LogDebug("Transforms: {Count}", pipeline.Transforms.Count);
+                        var transform = pipeline.Transforms[i];
+                        var configSummary = GetTransformConfigSummary(transform);
+                        logger.LogInformation("  {Index}. {Type}: {Summary}", i + 1, transform.TransformType, configSummary);
                    }
+                }

-                    if (pipeline.PostScripts.Count > 0)
+                if (pipeline.PostScripts.Count > 0)
+                {
+                    logger.LogInformation("");
+                    logger.LogInformation("Post-Scripts ({Count}):", pipeline.PostScripts.Count);
+                    for (int i = 0; i < pipeline.PostScripts.Count; i++)
                    {
-                        logger.LogDebug("");
-                        logger.LogDebug("Post-Scripts: {Count}", pipeline.PostScripts.Count);
+                        var script = pipeline.PostScripts[i];
+                        var scriptPreview = script.Script.Length > 50 ? script.Script[..50] + "..." : script.Script;
+                        scriptPreview = scriptPreview.Replace("\n", " ").Replace("\r", "");
+                        logger.LogInformation("  {Index}. [{Connection}] {Script}", i + 1, script.Connection, scriptPreview);
                    }
                }

@@ -512,6 +530,72 @@ public static class PipelineCommands
        return $"{minutes.Value}m";
    }

+    private static string GetTransformConfigSummary(TransformElement transform)
+    {
+        if (transform.Config == null || transform.Config.Value.ValueKind == System.Text.Json.JsonValueKind.Undefined)
+            return "(no config)";
+
+        try
+        {
+            var config = transform.Config.Value;
+
+            switch (transform.TransformType.ToLowerInvariant())
+            {
+                case "columndrop":
+                    if (config.TryGetProperty("Columns", out var columns) && columns.ValueKind == System.Text.Json.JsonValueKind.Array)
+                    {
+                        var columnList = new List<string>();
+                        foreach (var columnElement in columns.EnumerateArray())
+                        {
+                            columnList.Add(columnElement.GetString() ?? "");
+                        }
+                        return columnList.Count <= 3
+                            ? $"{columnList.Count} columns ({string.Join(", ", columnList)})"
+                            : $"{columnList.Count} columns ({string.Join(", ", columnList.Take(3))}, ...)";
+                    }
+                    break;
+
+                case "columnrename":
+                    if (config.TryGetProperty("Mappings", out var mappings) && mappings.ValueKind == System.Text.Json.JsonValueKind.Object)
+                    {
+                        var count = 0;
+                        foreach (var _ in mappings.EnumerateObject())
+                            count++;
+                        return $"{count} mappings";
+                    }
+                    break;
+
+                case "jdedate":
+                    var dateCol = config.TryGetProperty("DateColumn", out var d) ? d.GetString() : null;
+                    var timeCol = config.TryGetProperty("TimeColumn", out var t) ? t.GetString() : null;
+                    var outCol = config.TryGetProperty("OutputColumn", out var o) ? o.GetString() : null;
+                    if (!string.IsNullOrEmpty(dateCol) && !string.IsNullOrEmpty(outCol))
+                    {
+                        return !string.IsNullOrEmpty(timeCol)
+                            ? $"{dateCol} + {timeCol} -> {outCol}"
+                            : $"{dateCol} -> {outCol}";
+                    }
+                    break;
+
+                case "regex":
+                    var col = config.TryGetProperty("Column", out var c) ? c.GetString() : null;
+                    var pattern = config.TryGetProperty("Pattern", out var p) ? p.GetString() : null;
+                    if (!string.IsNullOrEmpty(col) && !string.IsNullOrEmpty(pattern))
+                    {
+                        var patternPreview = pattern.Length > 20 ? pattern[..20] + "..." : pattern;
+                        return $"{col}: /{patternPreview}/";
+                    }
+                    break;
+            }
+        }
+        catch
+        {
+            // Ignore JSON parsing errors
+        }
+
+        return "(configured)";
+    }
+
    private static async Task<string?> GetConfigFolderAsync(IServiceProvider serviceProvider, string? configPath)
    {
        if (!string.IsNullOrEmpty(configPath))
@@ -81,6 +81,16 @@ public static class Program
        pipelineCommand.AddCommand(PipelineCommands.CreateDeleteCommand(serviceProvider, configPathOption, verboseOption, quietOption));
        pipelineCommand.AddCommand(PipelineCommands.CreateEnableCommand(serviceProvider, configPathOption, verboseOption, quietOption));
        pipelineCommand.AddCommand(PipelineCommands.CreateDisableCommand(serviceProvider, configPathOption, verboseOption, quietOption));
+
+        // Pipeline element subcommand group
+        var elementCommand = new Command("element", "Manage pipeline elements");
+        elementCommand.AddCommand(PipelineElementCommands.CreatePreScriptCommand(serviceProvider, configPathOption, verboseOption, quietOption));
+        elementCommand.AddCommand(PipelineElementCommands.CreatePostScriptCommand(serviceProvider, configPathOption, verboseOption, quietOption));
+        elementCommand.AddCommand(PipelineElementCommands.CreateTransformCommand(serviceProvider, configPathOption, verboseOption, quietOption));
+        elementCommand.AddCommand(PipelineElementCommands.CreateSourceCommand(serviceProvider, configPathOption, verboseOption, quietOption));
+        elementCommand.AddCommand(PipelineElementCommands.CreateDestinationCommand(serviceProvider, configPathOption, verboseOption, quietOption));
+        pipelineCommand.AddCommand(elementCommand);
+
        rootCommand.AddCommand(pipelineCommand);

        // Config command group
@@ -125,6 +125,161 @@ jdescoping-config pipeline enable <name>
 jdescoping-config pipeline disable <name>
 ```

+### Pipeline Element Commands
+
+Manage elements within a pipeline (pre-scripts, transforms, post-scripts, source, destination).
+
+#### Pre-Script Management
+
+```bash
+# Add a pre-script
+jdescoping-config pipeline element prescript add WorkOrder_Curr \
+  --connection lotfinder \
+  --script "TRUNCATE TABLE dbo.WorkOrder_Temp;"
+
+# Add from a SQL file
+jdescoping-config pipeline element prescript add WorkOrder_Curr \
+  --connection lotfinder \
+  --script-file /path/to/script.sql
+
+# Add at a specific position (1-based index)
+jdescoping-config pipeline element prescript add WorkOrder_Curr \
+  --script "SELECT 1;" --at-index 1
+
+# Remove a pre-script (1-based index)
+jdescoping-config pipeline element prescript remove WorkOrder_Curr 2
+jdescoping-config pipeline element prescript remove WorkOrder_Curr 2 --force
+
+# Edit a pre-script
+jdescoping-config pipeline element prescript edit WorkOrder_Curr 1 \
+  --script "TRUNCATE TABLE dbo.NewTable;"
+
+# Move a pre-script up/down
+jdescoping-config pipeline element prescript move-up WorkOrder_Curr 3
+jdescoping-config pipeline element prescript move-down WorkOrder_Curr 1
+
+# Preview changes without saving
+jdescoping-config pipeline element prescript remove WorkOrder_Curr 2 --dry-run
+```
+
+#### Post-Script Management
+
+Post-scripts use the same commands as pre-scripts:
+
+```bash
+# Add a post-script
+jdescoping-config pipeline element postscript add WorkOrder_Curr \
+  --connection lotfinder \
+  --script "MERGE INTO dbo.WorkOrder_Hist..."
+
+# Remove, edit, and move work the same way
+jdescoping-config pipeline element postscript remove WorkOrder_Curr 1 --force
+jdescoping-config pipeline element postscript edit WorkOrder_Curr 1 --script "..."
+jdescoping-config pipeline element postscript move-up WorkOrder_Curr 2
+```
+
+#### Transform Management
+
+```bash
+# Add a ColumnDrop transform
+jdescoping-config pipeline element transform add WorkOrder_Curr \
+  --type ColumnDrop \
+  --columns "TempCol1,TempCol2,TempCol3"
+
+# Add a ColumnRename transform
+jdescoping-config pipeline element transform add WorkOrder_Curr \
+  --type ColumnRename \
+  --mappings "OldName=NewName,AnotherOld=AnotherNew"
+
+# Add a JdeDate transform
+jdescoping-config pipeline element transform add WorkOrder_Curr \
+  --type JdeDate \
+  --date-column "LastUpdateDate" \
+  --time-column "LastUpdateTime" \
+  --output-column "LastUpdateDT"
+
+# Add a Regex transform
+jdescoping-config pipeline element transform add WorkOrder_Curr \
+  --type Regex \
+  --column "Description" \
+  --pattern "^\s+" \
+  --replacement "" \
+  --ignore-case
+
+# Add with full JSON config
+jdescoping-config pipeline element transform add WorkOrder_Curr \
+  --type ColumnDrop \
+  --config '{"Columns":["Col1","Col2"]}'
+
+# Add from config file
+jdescoping-config pipeline element transform add WorkOrder_Curr \
+  --type JdeDate \
+  --config-file /path/to/transform-config.json
+
+# Remove a transform (1-based index)
+jdescoping-config pipeline element transform remove WorkOrder_Curr 2
+
+# Edit a transform
+jdescoping-config pipeline element transform edit WorkOrder_Curr 1 \
+  --columns "NewCol1,NewCol2"
+
+# Move transforms
+jdescoping-config pipeline element transform move-up WorkOrder_Curr 3
+jdescoping-config pipeline element transform move-down WorkOrder_Curr 1
+```
+
+#### Source Configuration
+
+```bash
+# Edit source connection
+jdescoping-config pipeline element source edit WorkOrder_Curr \
+  --connection jde
+
+# Edit source query
+jdescoping-config pipeline element source edit WorkOrder_Curr \
+  --query "SELECT * FROM F4801 WHERE WADOCO > :lastSync"
+
+# Edit from file
+jdescoping-config pipeline element source edit WorkOrder_Curr \
+  --query-file /path/to/query.sql
+
+# Edit mass sync query
+jdescoping-config pipeline element source edit WorkOrder_Curr \
+  --mass-query "SELECT * FROM F4801"
+
+# Edit multiple settings
+jdescoping-config pipeline element source edit WorkOrder_Curr \
+  --connection jde \
+  --query-file /path/to/incremental.sql \
+  --mass-query-file /path/to/mass.sql
+```
+
+#### Destination Configuration
+
+```bash
+# Edit destination table
+jdescoping-config pipeline element destination edit WorkOrder_Curr \
+  --table "dbo.WorkOrder_Curr"
+
+# Replace all match columns
+jdescoping-config pipeline element destination edit WorkOrder_Curr \
+  --match-columns "OrderNumber,LineNumber"
+
+# Add/remove individual match columns
+jdescoping-config pipeline element destination edit WorkOrder_Curr \
+  --add-match-column "NewColumn"
+jdescoping-config pipeline element destination edit WorkOrder_Curr \
+  --remove-match-column "OldColumn"
+
+# Manage exclude-from-update columns
+jdescoping-config pipeline element destination edit WorkOrder_Curr \
+  --exclude-from-update "CreatedDate,CreatedBy"
+jdescoping-config pipeline element destination edit WorkOrder_Curr \
+  --add-exclude "AuditColumn"
+jdescoping-config pipeline element destination edit WorkOrder_Curr \
+  --remove-exclude "OldAuditColumn"
+```
+
 ### Configuration Viewing Commands

 View configuration settings (read-only).