From 45f4ecab7dd7b35fb8cef603b61957967c9eef68 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 28 Jan 2026 16:08:28 -0500 Subject: [PATCH] feat(configmanager): add pipeline element management CLI commands Add commands to manage pipeline elements (pre-scripts, transforms, post-scripts, source, destination) matching the UI functionality: - prescript/postscript: add, remove, edit, move-up, move-down - transform: add, remove, edit, move-up, move-down with type shortcuts - source: edit (connection, query, mass-query) - destination: edit (table, match columns, exclude from update) Features include 1-based indices, --dry-run support, file input for scripts/queries, and numbered element display in pipeline show output. --- .../Commands/PipelineCommands.cs | 104 +- .../Commands/PipelineElementCommands.cs | 1979 +++++++++++++++++ .../JdeScoping.ConfigManager.Cli/Program.cs | 10 + .../JdeScoping.ConfigManager.Cli/README.md | 155 ++ .../Commands/PipelineElementCommandsTests.cs | 983 ++++++++ 5 files changed, 3221 insertions(+), 10 deletions(-) create mode 100644 NEW/src/Utils/JdeScoping.ConfigManager.Cli/Commands/PipelineElementCommands.cs create mode 100644 NEW/tests/Utils/JdeScoping.ConfigManager.Cli.Tests/Commands/PipelineElementCommandsTests.cs diff --git a/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Commands/PipelineCommands.cs b/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Commands/PipelineCommands.cs index 892edc8..2c3ea7a 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Commands/PipelineCommands.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Commands/PipelineCommands.cs @@ -283,24 +283,42 @@ public static class PipelineCommands logger.LogInformation(" Match Columns: {Value}", string.Join(", ", pipeline.Destination.MatchColumns)); } - if (verbose) + // Always show element counts and details when non-empty + if (pipeline.PreScripts.Count > 0) { - if (pipeline.PreScripts.Count > 0) + logger.LogInformation(""); + logger.LogInformation("Pre-Scripts ({Count}):", pipeline.PreScripts.Count); + for (int i = 0; i < pipeline.PreScripts.Count; i++) { - logger.LogDebug(""); - logger.LogDebug("Pre-Scripts: {Count}", pipeline.PreScripts.Count); + var script = pipeline.PreScripts[i]; + var scriptPreview = script.Script.Length > 50 ? script.Script[..50] + "..." : script.Script; + scriptPreview = scriptPreview.Replace("\n", " ").Replace("\r", ""); + logger.LogInformation(" {Index}. [{Connection}] {Script}", i + 1, script.Connection, scriptPreview); } + } - if (pipeline.Transforms.Count > 0) + if (pipeline.Transforms.Count > 0) + { + logger.LogInformation(""); + logger.LogInformation("Transforms ({Count}):", pipeline.Transforms.Count); + for (int i = 0; i < pipeline.Transforms.Count; i++) { - logger.LogDebug(""); - logger.LogDebug("Transforms: {Count}", pipeline.Transforms.Count); + var transform = pipeline.Transforms[i]; + var configSummary = GetTransformConfigSummary(transform); + logger.LogInformation(" {Index}. {Type}: {Summary}", i + 1, transform.TransformType, configSummary); } + } - if (pipeline.PostScripts.Count > 0) + if (pipeline.PostScripts.Count > 0) + { + logger.LogInformation(""); + logger.LogInformation("Post-Scripts ({Count}):", pipeline.PostScripts.Count); + for (int i = 0; i < pipeline.PostScripts.Count; i++) { - logger.LogDebug(""); - logger.LogDebug("Post-Scripts: {Count}", pipeline.PostScripts.Count); + var script = pipeline.PostScripts[i]; + var scriptPreview = script.Script.Length > 50 ? script.Script[..50] + "..." : script.Script; + scriptPreview = scriptPreview.Replace("\n", " ").Replace("\r", ""); + logger.LogInformation(" {Index}. [{Connection}] {Script}", i + 1, script.Connection, scriptPreview); } } @@ -512,6 +530,72 @@ public static class PipelineCommands return $"{minutes.Value}m"; } + private static string GetTransformConfigSummary(TransformElement transform) + { + if (transform.Config == null || transform.Config.Value.ValueKind == System.Text.Json.JsonValueKind.Undefined) + return "(no config)"; + + try + { + var config = transform.Config.Value; + + switch (transform.TransformType.ToLowerInvariant()) + { + case "columndrop": + if (config.TryGetProperty("Columns", out var columns) && columns.ValueKind == System.Text.Json.JsonValueKind.Array) + { + var columnList = new List(); + foreach (var columnElement in columns.EnumerateArray()) + { + columnList.Add(columnElement.GetString() ?? ""); + } + return columnList.Count <= 3 + ? $"{columnList.Count} columns ({string.Join(", ", columnList)})" + : $"{columnList.Count} columns ({string.Join(", ", columnList.Take(3))}, ...)"; + } + break; + + case "columnrename": + if (config.TryGetProperty("Mappings", out var mappings) && mappings.ValueKind == System.Text.Json.JsonValueKind.Object) + { + var count = 0; + foreach (var _ in mappings.EnumerateObject()) + count++; + return $"{count} mappings"; + } + break; + + case "jdedate": + var dateCol = config.TryGetProperty("DateColumn", out var d) ? d.GetString() : null; + var timeCol = config.TryGetProperty("TimeColumn", out var t) ? t.GetString() : null; + var outCol = config.TryGetProperty("OutputColumn", out var o) ? o.GetString() : null; + if (!string.IsNullOrEmpty(dateCol) && !string.IsNullOrEmpty(outCol)) + { + return !string.IsNullOrEmpty(timeCol) + ? $"{dateCol} + {timeCol} -> {outCol}" + : $"{dateCol} -> {outCol}"; + } + break; + + case "regex": + var col = config.TryGetProperty("Column", out var c) ? c.GetString() : null; + var pattern = config.TryGetProperty("Pattern", out var p) ? p.GetString() : null; + if (!string.IsNullOrEmpty(col) && !string.IsNullOrEmpty(pattern)) + { + var patternPreview = pattern.Length > 20 ? pattern[..20] + "..." : pattern; + return $"{col}: /{patternPreview}/"; + } + break; + } + } + catch + { + // Ignore JSON parsing errors + } + + return "(configured)"; + } + private static async Task GetConfigFolderAsync(IServiceProvider serviceProvider, string? configPath) { if (!string.IsNullOrEmpty(configPath)) diff --git a/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Commands/PipelineElementCommands.cs b/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Commands/PipelineElementCommands.cs new file mode 100644 index 0000000..f51104b --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Commands/PipelineElementCommands.cs @@ -0,0 +1,1979 @@ +using System.CommandLine; +using System.Text.Json; +using JdeScoping.ConfigManager.Core.Services; +using JdeScoping.DataSync.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace JdeScoping.ConfigManager.Cli.Commands; + +/// +/// Pipeline element management commands (pre-scripts, transforms, post-scripts, source, destination). +/// +public static class PipelineElementCommands +{ + private static readonly string[] ValidTransformTypes = ["ColumnDrop", "ColumnRename", "JdeDate", "Regex"]; + + #region PreScript Commands + + /// + /// Creates the prescript command group. + /// + public static Command CreatePreScriptCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("prescript", "Manage pipeline pre-scripts"); + + command.AddCommand(CreatePreScriptAddCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreatePreScriptRemoveCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreatePreScriptEditCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreatePreScriptMoveUpCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreatePreScriptMoveDownCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + + return command; + } + + private static Command CreatePreScriptAddCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("add", "Add a pre-script to a pipeline"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + command.AddArgument(pipelineArg); + + var connectionOption = new Option( + aliases: ["--connection", "-c"], + getDefaultValue: () => "lotfinder", + description: "Connection name for script execution"); + command.AddOption(connectionOption); + + var scriptOption = new Option( + aliases: ["--script", "-s"], + description: "SQL script text"); + command.AddOption(scriptOption); + + var scriptFileOption = new Option( + aliases: ["--script-file", "-f"], + description: "Path to SQL file"); + command.AddOption(scriptFileOption); + + var atIndexOption = new Option( + aliases: ["--at-index", "-i"], + description: "Position to insert (1-based, default: end)"); + command.AddOption(atIndexOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, string connection, string? script, string? scriptFile, int? atIndex) => + { + var exitCode = await AddScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, connection, script, scriptFile, atIndex, isPreScript: true); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, connectionOption, scriptOption, scriptFileOption, atIndexOption); + + return command; + } + + private static Command CreatePreScriptRemoveCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("remove", "Remove a pre-script from a pipeline"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The pre-script index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var forceOption = new Option( + aliases: ["--force", "-f"], + description: "Skip confirmation prompt"); + command.AddOption(forceOption); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, int index, bool force, bool dryRun) => + { + var exitCode = await RemoveScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, force, dryRun, isPreScript: true); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, indexArg, forceOption, dryRunOption); + + return command; + } + + private static Command CreatePreScriptEditCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("edit", "Edit a pre-script in a pipeline"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The pre-script index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var connectionOption = new Option( + aliases: ["--connection", "-c"], + description: "Connection name for script execution"); + command.AddOption(connectionOption); + + var scriptOption = new Option( + aliases: ["--script", "-s"], + description: "SQL script text"); + command.AddOption(scriptOption); + + var scriptFileOption = new Option( + aliases: ["--script-file", "-f"], + description: "Path to SQL file"); + command.AddOption(scriptFileOption); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async context => + { + var configPath = context.ParseResult.GetValueForOption(configPathOption); + var verbose = context.ParseResult.GetValueForOption(verboseOption); + var quiet = context.ParseResult.GetValueForOption(quietOption); + var pipelineName = context.ParseResult.GetValueForArgument(pipelineArg); + var index = context.ParseResult.GetValueForArgument(indexArg); + var connection = context.ParseResult.GetValueForOption(connectionOption); + var script = context.ParseResult.GetValueForOption(scriptOption); + var scriptFile = context.ParseResult.GetValueForOption(scriptFileOption); + var dryRun = context.ParseResult.GetValueForOption(dryRunOption); + + var exitCode = await EditScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, connection, script, scriptFile, dryRun, isPreScript: true); + Environment.ExitCode = exitCode; + }); + + return command; + } + + private static Command CreatePreScriptMoveUpCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("move-up", "Move a pre-script up in the list"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The pre-script index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, int index, bool dryRun) => + { + var exitCode = await MoveScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, dryRun, isPreScript: true, moveUp: true); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, indexArg, dryRunOption); + + return command; + } + + private static Command CreatePreScriptMoveDownCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("move-down", "Move a pre-script down in the list"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The pre-script index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, int index, bool dryRun) => + { + var exitCode = await MoveScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, dryRun, isPreScript: true, moveUp: false); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, indexArg, dryRunOption); + + return command; + } + + #endregion + + #region PostScript Commands + + /// + /// Creates the postscript command group. + /// + public static Command CreatePostScriptCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("postscript", "Manage pipeline post-scripts"); + + command.AddCommand(CreatePostScriptAddCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreatePostScriptRemoveCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreatePostScriptEditCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreatePostScriptMoveUpCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreatePostScriptMoveDownCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + + return command; + } + + private static Command CreatePostScriptAddCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("add", "Add a post-script to a pipeline"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + command.AddArgument(pipelineArg); + + var connectionOption = new Option( + aliases: ["--connection", "-c"], + getDefaultValue: () => "lotfinder", + description: "Connection name for script execution"); + command.AddOption(connectionOption); + + var scriptOption = new Option( + aliases: ["--script", "-s"], + description: "SQL script text"); + command.AddOption(scriptOption); + + var scriptFileOption = new Option( + aliases: ["--script-file", "-f"], + description: "Path to SQL file"); + command.AddOption(scriptFileOption); + + var atIndexOption = new Option( + aliases: ["--at-index", "-i"], + description: "Position to insert (1-based, default: end)"); + command.AddOption(atIndexOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, string connection, string? script, string? scriptFile, int? atIndex) => + { + var exitCode = await AddScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, connection, script, scriptFile, atIndex, isPreScript: false); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, connectionOption, scriptOption, scriptFileOption, atIndexOption); + + return command; + } + + private static Command CreatePostScriptRemoveCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("remove", "Remove a post-script from a pipeline"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The post-script index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var forceOption = new Option( + aliases: ["--force", "-f"], + description: "Skip confirmation prompt"); + command.AddOption(forceOption); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, int index, bool force, bool dryRun) => + { + var exitCode = await RemoveScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, force, dryRun, isPreScript: false); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, indexArg, forceOption, dryRunOption); + + return command; + } + + private static Command CreatePostScriptEditCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("edit", "Edit a post-script in a pipeline"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The post-script index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var connectionOption = new Option( + aliases: ["--connection", "-c"], + description: "Connection name for script execution"); + command.AddOption(connectionOption); + + var scriptOption = new Option( + aliases: ["--script", "-s"], + description: "SQL script text"); + command.AddOption(scriptOption); + + var scriptFileOption = new Option( + aliases: ["--script-file", "-f"], + description: "Path to SQL file"); + command.AddOption(scriptFileOption); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async context => + { + var configPath = context.ParseResult.GetValueForOption(configPathOption); + var verbose = context.ParseResult.GetValueForOption(verboseOption); + var quiet = context.ParseResult.GetValueForOption(quietOption); + var pipelineName = context.ParseResult.GetValueForArgument(pipelineArg); + var index = context.ParseResult.GetValueForArgument(indexArg); + var connection = context.ParseResult.GetValueForOption(connectionOption); + var script = context.ParseResult.GetValueForOption(scriptOption); + var scriptFile = context.ParseResult.GetValueForOption(scriptFileOption); + var dryRun = context.ParseResult.GetValueForOption(dryRunOption); + + var exitCode = await EditScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, connection, script, scriptFile, dryRun, isPreScript: false); + Environment.ExitCode = exitCode; + }); + + return command; + } + + private static Command CreatePostScriptMoveUpCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("move-up", "Move a post-script up in the list"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The post-script index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, int index, bool dryRun) => + { + var exitCode = await MoveScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, dryRun, isPreScript: false, moveUp: true); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, indexArg, dryRunOption); + + return command; + } + + private static Command CreatePostScriptMoveDownCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("move-down", "Move a post-script down in the list"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The post-script index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, int index, bool dryRun) => + { + var exitCode = await MoveScriptAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, dryRun, isPreScript: false, moveUp: false); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, indexArg, dryRunOption); + + return command; + } + + #endregion + + #region Transform Commands + + /// + /// Creates the transform command group. + /// + public static Command CreateTransformCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("transform", "Manage pipeline transforms"); + + command.AddCommand(CreateTransformAddCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreateTransformRemoveCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreateTransformEditCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreateTransformMoveUpCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + command.AddCommand(CreateTransformMoveDownCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + + return command; + } + + private static Command CreateTransformAddCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("add", "Add a transform to a pipeline"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + command.AddArgument(pipelineArg); + + var typeOption = new Option( + aliases: ["--type", "-t"], + description: "Transform type (ColumnDrop, ColumnRename, JdeDate, Regex)") + { IsRequired = true }; + command.AddOption(typeOption); + + var configOption = new Option( + aliases: ["--config", "-c"], + description: "Full config as JSON"); + command.AddOption(configOption); + + var configFileOption = new Option( + aliases: ["--config-file"], + description: "Config from file"); + command.AddOption(configFileOption); + + var atIndexOption = new Option( + aliases: ["--at-index", "-i"], + description: "Position to insert (1-based, default: end)"); + command.AddOption(atIndexOption); + + // Type-specific shortcuts + var columnsOption = new Option( + aliases: ["--columns"], + description: "Columns for ColumnDrop (comma-separated)"); + command.AddOption(columnsOption); + + var mappingsOption = new Option( + aliases: ["--mappings"], + description: "Mappings for ColumnRename (old=new,old2=new2)"); + command.AddOption(mappingsOption); + + var dateColumnOption = new Option( + aliases: ["--date-column"], + description: "Date column for JdeDate"); + command.AddOption(dateColumnOption); + + var timeColumnOption = new Option( + aliases: ["--time-column"], + description: "Time column for JdeDate"); + command.AddOption(timeColumnOption); + + var outputColumnOption = new Option( + aliases: ["--output-column"], + description: "Output column for JdeDate"); + command.AddOption(outputColumnOption); + + var columnOption = new Option( + aliases: ["--column"], + description: "Column for Regex"); + command.AddOption(columnOption); + + var patternOption = new Option( + aliases: ["--pattern"], + description: "Pattern for Regex"); + command.AddOption(patternOption); + + var replacementOption = new Option( + aliases: ["--replacement"], + description: "Replacement for Regex"); + command.AddOption(replacementOption); + + var ignoreCaseOption = new Option( + aliases: ["--ignore-case"], + description: "Ignore case for Regex"); + command.AddOption(ignoreCaseOption); + + command.SetHandler(async context => + { + var configPath = context.ParseResult.GetValueForOption(configPathOption); + var verbose = context.ParseResult.GetValueForOption(verboseOption); + var quiet = context.ParseResult.GetValueForOption(quietOption); + var pipelineName = context.ParseResult.GetValueForArgument(pipelineArg); + var type = context.ParseResult.GetValueForOption(typeOption)!; + var config = context.ParseResult.GetValueForOption(configOption); + var configFile = context.ParseResult.GetValueForOption(configFileOption); + var atIndex = context.ParseResult.GetValueForOption(atIndexOption); + var columns = context.ParseResult.GetValueForOption(columnsOption); + var mappings = context.ParseResult.GetValueForOption(mappingsOption); + var dateColumn = context.ParseResult.GetValueForOption(dateColumnOption); + var timeColumn = context.ParseResult.GetValueForOption(timeColumnOption); + var outputColumn = context.ParseResult.GetValueForOption(outputColumnOption); + var column = context.ParseResult.GetValueForOption(columnOption); + var pattern = context.ParseResult.GetValueForOption(patternOption); + var replacement = context.ParseResult.GetValueForOption(replacementOption); + var ignoreCase = context.ParseResult.GetValueForOption(ignoreCaseOption); + + var exitCode = await AddTransformAsync(serviceProvider, configPath, verbose, quiet, pipelineName, type, + config, configFile, atIndex, columns, mappings, dateColumn, timeColumn, outputColumn, + column, pattern, replacement, ignoreCase); + Environment.ExitCode = exitCode; + }); + + return command; + } + + private static Command CreateTransformRemoveCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("remove", "Remove a transform from a pipeline"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The transform index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var forceOption = new Option( + aliases: ["--force", "-f"], + description: "Skip confirmation prompt"); + command.AddOption(forceOption); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, int index, bool force, bool dryRun) => + { + var exitCode = await RemoveTransformAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, force, dryRun); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, indexArg, forceOption, dryRunOption); + + return command; + } + + private static Command CreateTransformEditCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("edit", "Edit a transform in a pipeline"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The transform index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var typeOption = new Option( + aliases: ["--type", "-t"], + description: "Transform type (ColumnDrop, ColumnRename, JdeDate, Regex)"); + command.AddOption(typeOption); + + var configOption = new Option( + aliases: ["--config", "-c"], + description: "Full config as JSON"); + command.AddOption(configOption); + + var configFileOption = new Option( + aliases: ["--config-file"], + description: "Config from file"); + command.AddOption(configFileOption); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + // Type-specific shortcuts + var columnsOption = new Option( + aliases: ["--columns"], + description: "Columns for ColumnDrop (comma-separated)"); + command.AddOption(columnsOption); + + var mappingsOption = new Option( + aliases: ["--mappings"], + description: "Mappings for ColumnRename (old=new,old2=new2)"); + command.AddOption(mappingsOption); + + var dateColumnOption = new Option( + aliases: ["--date-column"], + description: "Date column for JdeDate"); + command.AddOption(dateColumnOption); + + var timeColumnOption = new Option( + aliases: ["--time-column"], + description: "Time column for JdeDate"); + command.AddOption(timeColumnOption); + + var outputColumnOption = new Option( + aliases: ["--output-column"], + description: "Output column for JdeDate"); + command.AddOption(outputColumnOption); + + var columnOption = new Option( + aliases: ["--column"], + description: "Column for Regex"); + command.AddOption(columnOption); + + var patternOption = new Option( + aliases: ["--pattern"], + description: "Pattern for Regex"); + command.AddOption(patternOption); + + var replacementOption = new Option( + aliases: ["--replacement"], + description: "Replacement for Regex"); + command.AddOption(replacementOption); + + var ignoreCaseOption = new Option( + aliases: ["--ignore-case"], + description: "Ignore case for Regex"); + command.AddOption(ignoreCaseOption); + + command.SetHandler(async context => + { + var configPath = context.ParseResult.GetValueForOption(configPathOption); + var verbose = context.ParseResult.GetValueForOption(verboseOption); + var quiet = context.ParseResult.GetValueForOption(quietOption); + var pipelineName = context.ParseResult.GetValueForArgument(pipelineArg); + var index = context.ParseResult.GetValueForArgument(indexArg); + var type = context.ParseResult.GetValueForOption(typeOption); + var config = context.ParseResult.GetValueForOption(configOption); + var configFile = context.ParseResult.GetValueForOption(configFileOption); + var dryRun = context.ParseResult.GetValueForOption(dryRunOption); + var columns = context.ParseResult.GetValueForOption(columnsOption); + var mappings = context.ParseResult.GetValueForOption(mappingsOption); + var dateColumn = context.ParseResult.GetValueForOption(dateColumnOption); + var timeColumn = context.ParseResult.GetValueForOption(timeColumnOption); + var outputColumn = context.ParseResult.GetValueForOption(outputColumnOption); + var column = context.ParseResult.GetValueForOption(columnOption); + var pattern = context.ParseResult.GetValueForOption(patternOption); + var replacement = context.ParseResult.GetValueForOption(replacementOption); + var ignoreCase = context.ParseResult.GetValueForOption(ignoreCaseOption); + + var exitCode = await EditTransformAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, type, + config, configFile, dryRun, columns, mappings, dateColumn, timeColumn, outputColumn, + column, pattern, replacement, ignoreCase); + Environment.ExitCode = exitCode; + }); + + return command; + } + + private static Command CreateTransformMoveUpCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("move-up", "Move a transform up in the list"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The transform index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, int index, bool dryRun) => + { + var exitCode = await MoveTransformAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, dryRun, moveUp: true); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, indexArg, dryRunOption); + + return command; + } + + private static Command CreateTransformMoveDownCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("move-down", "Move a transform down in the list"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + var indexArg = new Argument("index", "The transform index (1-based)"); + command.AddArgument(pipelineArg); + command.AddArgument(indexArg); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async (string? configPath, bool verbose, bool quiet, string pipelineName, int index, bool dryRun) => + { + var exitCode = await MoveTransformAsync(serviceProvider, configPath, verbose, quiet, pipelineName, index, dryRun, moveUp: false); + Environment.ExitCode = exitCode; + }, configPathOption, verboseOption, quietOption, pipelineArg, indexArg, dryRunOption); + + return command; + } + + #endregion + + #region Source Commands + + /// + /// Creates the source command group. + /// + public static Command CreateSourceCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("source", "Manage pipeline source"); + + command.AddCommand(CreateSourceEditCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + + return command; + } + + private static Command CreateSourceEditCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("edit", "Edit pipeline source configuration"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + command.AddArgument(pipelineArg); + + var connectionOption = new Option( + aliases: ["--connection", "-c"], + description: "Connection name"); + command.AddOption(connectionOption); + + var queryOption = new Option( + aliases: ["--query", "-q"], + description: "Incremental sync query"); + command.AddOption(queryOption); + + var queryFileOption = new Option( + aliases: ["--query-file"], + description: "Query from file"); + command.AddOption(queryFileOption); + + var massQueryOption = new Option( + aliases: ["--mass-query"], + description: "Mass sync query"); + command.AddOption(massQueryOption); + + var massQueryFileOption = new Option( + aliases: ["--mass-query-file"], + description: "Mass query from file"); + command.AddOption(massQueryFileOption); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async context => + { + var configPath = context.ParseResult.GetValueForOption(configPathOption); + var verbose = context.ParseResult.GetValueForOption(verboseOption); + var quiet = context.ParseResult.GetValueForOption(quietOption); + var pipelineName = context.ParseResult.GetValueForArgument(pipelineArg); + var connection = context.ParseResult.GetValueForOption(connectionOption); + var query = context.ParseResult.GetValueForOption(queryOption); + var queryFile = context.ParseResult.GetValueForOption(queryFileOption); + var massQuery = context.ParseResult.GetValueForOption(massQueryOption); + var massQueryFile = context.ParseResult.GetValueForOption(massQueryFileOption); + var dryRun = context.ParseResult.GetValueForOption(dryRunOption); + + var exitCode = await EditSourceAsync(serviceProvider, configPath, verbose, quiet, pipelineName, connection, query, queryFile, massQuery, massQueryFile, dryRun); + Environment.ExitCode = exitCode; + }); + + return command; + } + + #endregion + + #region Destination Commands + + /// + /// Creates the destination command group. + /// + public static Command CreateDestinationCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("destination", "Manage pipeline destination"); + + command.AddCommand(CreateDestinationEditCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + + return command; + } + + private static Command CreateDestinationEditCommand( + IServiceProvider serviceProvider, + Option configPathOption, + Option verboseOption, + Option quietOption) + { + var command = new Command("edit", "Edit pipeline destination configuration"); + + var pipelineArg = new Argument("pipeline-name", "The pipeline name"); + command.AddArgument(pipelineArg); + + var tableOption = new Option( + aliases: ["--table", "-t"], + description: "Table name"); + command.AddOption(tableOption); + + var matchColumnsOption = new Option( + aliases: ["--match-columns"], + description: "Match columns (comma-separated, replaces all)"); + command.AddOption(matchColumnsOption); + + var addMatchColumnOption = new Option( + aliases: ["--add-match-column"], + description: "Add single match column"); + command.AddOption(addMatchColumnOption); + + var removeMatchColumnOption = new Option( + aliases: ["--remove-match-column"], + description: "Remove single match column"); + command.AddOption(removeMatchColumnOption); + + var excludeFromUpdateOption = new Option( + aliases: ["--exclude-from-update"], + description: "Exclude columns (comma-separated, replaces all)"); + command.AddOption(excludeFromUpdateOption); + + var addExcludeOption = new Option( + aliases: ["--add-exclude"], + description: "Add to exclude list"); + command.AddOption(addExcludeOption); + + var removeExcludeOption = new Option( + aliases: ["--remove-exclude"], + description: "Remove from exclude list"); + command.AddOption(removeExcludeOption); + + var dryRunOption = new Option( + aliases: ["--dry-run"], + description: "Preview changes without saving"); + command.AddOption(dryRunOption); + + command.SetHandler(async context => + { + var configPath = context.ParseResult.GetValueForOption(configPathOption); + var verbose = context.ParseResult.GetValueForOption(verboseOption); + var quiet = context.ParseResult.GetValueForOption(quietOption); + var pipelineName = context.ParseResult.GetValueForArgument(pipelineArg); + var table = context.ParseResult.GetValueForOption(tableOption); + var matchColumns = context.ParseResult.GetValueForOption(matchColumnsOption); + var addMatchColumn = context.ParseResult.GetValueForOption(addMatchColumnOption); + var removeMatchColumn = context.ParseResult.GetValueForOption(removeMatchColumnOption); + var excludeFromUpdate = context.ParseResult.GetValueForOption(excludeFromUpdateOption); + var addExclude = context.ParseResult.GetValueForOption(addExcludeOption); + var removeExclude = context.ParseResult.GetValueForOption(removeExcludeOption); + var dryRun = context.ParseResult.GetValueForOption(dryRunOption); + + var exitCode = await EditDestinationAsync(serviceProvider, configPath, verbose, quiet, pipelineName, + table, matchColumns, addMatchColumn, removeMatchColumn, excludeFromUpdate, addExclude, removeExclude, dryRun); + Environment.ExitCode = exitCode; + }); + + return command; + } + + #endregion + + #region Handler Implementations + + private static async Task AddScriptAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + string connection, + string? script, + string? scriptFile, + int? atIndex, + bool isPreScript) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + var scriptType = isPreScript ? "pre-script" : "post-script"; + + using (logger.BeginScope(new Dictionary + { + ["Command"] = $"pipeline element {(isPreScript ? "prescript" : "postscript")} add", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + // Validate script input + if (string.IsNullOrEmpty(script) && string.IsNullOrEmpty(scriptFile)) + { + logger.LogError("Either --script or --script-file must be provided"); + return 1; + } + + if (!string.IsNullOrEmpty(script) && !string.IsNullOrEmpty(scriptFile)) + { + logger.LogError("Cannot specify both --script and --script-file"); + return 1; + } + + // Load script from file if needed + var scriptText = script; + if (!string.IsNullOrEmpty(scriptFile)) + { + if (!File.Exists(scriptFile)) + { + logger.LogError("File not found: {Path}", scriptFile); + return 1; + } + scriptText = await File.ReadAllTextAsync(scriptFile); + } + + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + var scripts = isPreScript ? pipeline.PreScripts : pipeline.PostScripts; + var newScript = new ScriptElement + { + Connection = connection, + Script = scriptText! + }; + + // Determine insert position + int insertIndex = scripts.Count; + if (atIndex.HasValue) + { + if (atIndex.Value < 1 || atIndex.Value > scripts.Count + 1) + { + logger.LogError("Index {Index} is out of range. Valid range: 1 to {Max}", atIndex.Value, scripts.Count + 1); + return 1; + } + insertIndex = atIndex.Value - 1; + } + + scripts.Insert(insertIndex, newScript); + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Added {ScriptType} at position {Position} to pipeline '{Name}'", scriptType, insertIndex + 1, pipelineName); + + return 0; + } + } + + private static async Task RemoveScriptAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + int index, + bool force, + bool dryRun, + bool isPreScript) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + var scriptType = isPreScript ? "pre-script" : "post-script"; + + using (logger.BeginScope(new Dictionary + { + ["Command"] = $"pipeline element {(isPreScript ? "prescript" : "postscript")} remove", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + var scripts = isPreScript ? pipeline.PreScripts : pipeline.PostScripts; + + // Validate index (1-based input) + var zeroBasedIndex = index - 1; + if (zeroBasedIndex < 0 || zeroBasedIndex >= scripts.Count) + { + logger.LogError("Index {Index} is out of range. Pipeline has {Count} {ScriptType}s", index, scripts.Count, scriptType); + return 1; + } + + var scriptToRemove = scripts[zeroBasedIndex]; + + if (dryRun) + { + logger.LogInformation("[DRY RUN] Would remove {ScriptType} {Index}: [{Connection}] {Script}", + scriptType, index, scriptToRemove.Connection, + scriptToRemove.Script.Length > 50 ? scriptToRemove.Script[..50] + "..." : scriptToRemove.Script); + return 0; + } + + if (!force) + { + Console.Write($"Remove {scriptType} {index} from '{pipelineName}'? [y/N] "); + var response = Console.ReadLine(); + if (!string.Equals(response, "y", StringComparison.OrdinalIgnoreCase) && + !string.Equals(response, "yes", StringComparison.OrdinalIgnoreCase)) + { + logger.LogInformation("Cancelled"); + return 0; + } + } + + scripts.RemoveAt(zeroBasedIndex); + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Removed {ScriptType} {Index} from pipeline '{Name}'", scriptType, index, pipelineName); + + return 0; + } + } + + private static async Task EditScriptAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + int index, + string? connection, + string? script, + string? scriptFile, + bool dryRun, + bool isPreScript) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + var scriptType = isPreScript ? "pre-script" : "post-script"; + + using (logger.BeginScope(new Dictionary + { + ["Command"] = $"pipeline element {(isPreScript ? "prescript" : "postscript")} edit", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + // Validate script input + if (!string.IsNullOrEmpty(script) && !string.IsNullOrEmpty(scriptFile)) + { + logger.LogError("Cannot specify both --script and --script-file"); + return 1; + } + + // Load script from file if needed + var scriptText = script; + if (!string.IsNullOrEmpty(scriptFile)) + { + if (!File.Exists(scriptFile)) + { + logger.LogError("File not found: {Path}", scriptFile); + return 1; + } + scriptText = await File.ReadAllTextAsync(scriptFile); + } + + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + var scripts = isPreScript ? pipeline.PreScripts : pipeline.PostScripts; + + // Validate index (1-based input) + var zeroBasedIndex = index - 1; + if (zeroBasedIndex < 0 || zeroBasedIndex >= scripts.Count) + { + logger.LogError("Index {Index} is out of range. Pipeline has {Count} {ScriptType}s", index, scripts.Count, scriptType); + return 1; + } + + var scriptToEdit = scripts[zeroBasedIndex]; + var changes = new List(); + + if (!string.IsNullOrEmpty(connection)) + { + changes.Add($"Connection: {scriptToEdit.Connection} -> {connection}"); + if (!dryRun) scriptToEdit.Connection = connection; + } + + if (!string.IsNullOrEmpty(scriptText)) + { + changes.Add("Script: (updated)"); + if (!dryRun) scriptToEdit.Script = scriptText; + } + + if (changes.Count == 0) + { + logger.LogWarning("No changes specified"); + return 0; + } + + if (dryRun) + { + logger.LogInformation("[DRY RUN] Would update {ScriptType} {Index}:", scriptType, index); + foreach (var change in changes) + { + logger.LogInformation(" {Change}", change); + } + return 0; + } + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Updated {ScriptType} {Index} in pipeline '{Name}'", scriptType, index, pipelineName); + + return 0; + } + } + + private static async Task MoveScriptAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + int index, + bool dryRun, + bool isPreScript, + bool moveUp) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + var scriptType = isPreScript ? "pre-script" : "post-script"; + var direction = moveUp ? "up" : "down"; + + using (logger.BeginScope(new Dictionary + { + ["Command"] = $"pipeline element {(isPreScript ? "prescript" : "postscript")} move-{direction}", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + var scripts = isPreScript ? pipeline.PreScripts : pipeline.PostScripts; + + // Validate index (1-based input) + var zeroBasedIndex = index - 1; + if (zeroBasedIndex < 0 || zeroBasedIndex >= scripts.Count) + { + logger.LogError("Index {Index} is out of range. Pipeline has {Count} {ScriptType}s", index, scripts.Count, scriptType); + return 1; + } + + // Check boundary conditions + if (moveUp && zeroBasedIndex == 0) + { + logger.LogWarning("{ScriptType} is already at the first position", char.ToUpper(scriptType[0]) + scriptType[1..]); + return 0; + } + + if (!moveUp && zeroBasedIndex == scripts.Count - 1) + { + logger.LogWarning("{ScriptType} is already at the last position", char.ToUpper(scriptType[0]) + scriptType[1..]); + return 0; + } + + var swapIndex = moveUp ? zeroBasedIndex - 1 : zeroBasedIndex + 1; + + if (dryRun) + { + logger.LogInformation("[DRY RUN] Would swap {ScriptType} {Index1} with {ScriptType} {Index2}", + scriptType, index, scriptType, swapIndex + 1); + return 0; + } + + // Swap elements + (scripts[zeroBasedIndex], scripts[swapIndex]) = (scripts[swapIndex], scripts[zeroBasedIndex]); + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Moved {ScriptType} {OldIndex} to position {NewIndex} in pipeline '{Name}'", + scriptType, index, swapIndex + 1, pipelineName); + + return 0; + } + } + + private static async Task AddTransformAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + string type, + string? config, + string? configFile, + int? atIndex, + string? columns, + string? mappings, + string? dateColumn, + string? timeColumn, + string? outputColumn, + string? column, + string? pattern, + string? replacement, + bool? ignoreCase) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + + using (logger.BeginScope(new Dictionary + { + ["Command"] = "pipeline element transform add", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + // Validate transform type + if (!ValidTransformTypes.Contains(type, StringComparer.OrdinalIgnoreCase)) + { + logger.LogError("Unknown transform type '{Type}'. Valid types: {ValidTypes}", type, string.Join(", ", ValidTransformTypes)); + return 1; + } + + // Build config JSON + JsonElement? transformConfig; + try + { + transformConfig = await BuildTransformConfigAsync(type, config, configFile, columns, mappings, + dateColumn, timeColumn, outputColumn, column, pattern, replacement, ignoreCase, logger); + } + catch (Exception ex) + { + logger.LogError("Invalid configuration: {Message}", ex.Message); + return 1; + } + + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + var newTransform = new TransformElement + { + TransformType = type, + Config = transformConfig + }; + + // Determine insert position + int insertIndex = pipeline.Transforms.Count; + if (atIndex.HasValue) + { + if (atIndex.Value < 1 || atIndex.Value > pipeline.Transforms.Count + 1) + { + logger.LogError("Index {Index} is out of range. Valid range: 1 to {Max}", atIndex.Value, pipeline.Transforms.Count + 1); + return 1; + } + insertIndex = atIndex.Value - 1; + } + + pipeline.Transforms.Insert(insertIndex, newTransform); + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Added transform '{Type}' at position {Position} to pipeline '{Name}'", type, insertIndex + 1, pipelineName); + + return 0; + } + } + + private static async Task RemoveTransformAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + int index, + bool force, + bool dryRun) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + + using (logger.BeginScope(new Dictionary + { + ["Command"] = "pipeline element transform remove", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + // Validate index (1-based input) + var zeroBasedIndex = index - 1; + if (zeroBasedIndex < 0 || zeroBasedIndex >= pipeline.Transforms.Count) + { + logger.LogError("Index {Index} is out of range. Pipeline has {Count} transforms", index, pipeline.Transforms.Count); + return 1; + } + + var transformToRemove = pipeline.Transforms[zeroBasedIndex]; + + if (dryRun) + { + logger.LogInformation("[DRY RUN] Would remove transform {Index}: {Type}", index, transformToRemove.TransformType); + return 0; + } + + if (!force) + { + Console.Write($"Remove transform {index} ({transformToRemove.TransformType}) from '{pipelineName}'? [y/N] "); + var response = Console.ReadLine(); + if (!string.Equals(response, "y", StringComparison.OrdinalIgnoreCase) && + !string.Equals(response, "yes", StringComparison.OrdinalIgnoreCase)) + { + logger.LogInformation("Cancelled"); + return 0; + } + } + + pipeline.Transforms.RemoveAt(zeroBasedIndex); + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Removed transform {Index} from pipeline '{Name}'", index, pipelineName); + + return 0; + } + } + + private static async Task EditTransformAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + int index, + string? type, + string? config, + string? configFile, + bool dryRun, + string? columns, + string? mappings, + string? dateColumn, + string? timeColumn, + string? outputColumn, + string? column, + string? pattern, + string? replacement, + bool? ignoreCase) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + + using (logger.BeginScope(new Dictionary + { + ["Command"] = "pipeline element transform edit", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + // Validate transform type if provided + if (!string.IsNullOrEmpty(type) && !ValidTransformTypes.Contains(type, StringComparer.OrdinalIgnoreCase)) + { + logger.LogError("Unknown transform type '{Type}'. Valid types: {ValidTypes}", type, string.Join(", ", ValidTransformTypes)); + return 1; + } + + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + // Validate index (1-based input) + var zeroBasedIndex = index - 1; + if (zeroBasedIndex < 0 || zeroBasedIndex >= pipeline.Transforms.Count) + { + logger.LogError("Index {Index} is out of range. Pipeline has {Count} transforms", index, pipeline.Transforms.Count); + return 1; + } + + var transformToEdit = pipeline.Transforms[zeroBasedIndex]; + var changes = new List(); + + // Determine the effective type for config building + var effectiveType = type ?? transformToEdit.TransformType; + + if (!string.IsNullOrEmpty(type)) + { + changes.Add($"Type: {transformToEdit.TransformType} -> {type}"); + if (!dryRun) transformToEdit.TransformType = type; + } + + // Check if any config-related options were provided + var hasConfigOptions = !string.IsNullOrEmpty(config) || !string.IsNullOrEmpty(configFile) || + !string.IsNullOrEmpty(columns) || !string.IsNullOrEmpty(mappings) || + !string.IsNullOrEmpty(dateColumn) || !string.IsNullOrEmpty(timeColumn) || + !string.IsNullOrEmpty(outputColumn) || !string.IsNullOrEmpty(column) || + !string.IsNullOrEmpty(pattern) || !string.IsNullOrEmpty(replacement) || + ignoreCase.HasValue; + + if (hasConfigOptions) + { + try + { + var newConfig = await BuildTransformConfigAsync(effectiveType, config, configFile, columns, mappings, + dateColumn, timeColumn, outputColumn, column, pattern, replacement, ignoreCase, logger); + changes.Add("Config: (updated)"); + if (!dryRun) transformToEdit.Config = newConfig; + } + catch (Exception ex) + { + logger.LogError("Invalid configuration: {Message}", ex.Message); + return 1; + } + } + + if (changes.Count == 0) + { + logger.LogWarning("No changes specified"); + return 0; + } + + if (dryRun) + { + logger.LogInformation("[DRY RUN] Would update transform {Index}:", index); + foreach (var change in changes) + { + logger.LogInformation(" {Change}", change); + } + return 0; + } + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Updated transform {Index} in pipeline '{Name}'", index, pipelineName); + + return 0; + } + } + + private static async Task MoveTransformAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + int index, + bool dryRun, + bool moveUp) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + var direction = moveUp ? "up" : "down"; + + using (logger.BeginScope(new Dictionary + { + ["Command"] = $"pipeline element transform move-{direction}", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + // Validate index (1-based input) + var zeroBasedIndex = index - 1; + if (zeroBasedIndex < 0 || zeroBasedIndex >= pipeline.Transforms.Count) + { + logger.LogError("Index {Index} is out of range. Pipeline has {Count} transforms", index, pipeline.Transforms.Count); + return 1; + } + + // Check boundary conditions + if (moveUp && zeroBasedIndex == 0) + { + logger.LogWarning("Transform is already at the first position"); + return 0; + } + + if (!moveUp && zeroBasedIndex == pipeline.Transforms.Count - 1) + { + logger.LogWarning("Transform is already at the last position"); + return 0; + } + + var swapIndex = moveUp ? zeroBasedIndex - 1 : zeroBasedIndex + 1; + + if (dryRun) + { + logger.LogInformation("[DRY RUN] Would swap transform {Index1} with transform {Index2}", + index, swapIndex + 1); + return 0; + } + + // Swap elements + (pipeline.Transforms[zeroBasedIndex], pipeline.Transforms[swapIndex]) = + (pipeline.Transforms[swapIndex], pipeline.Transforms[zeroBasedIndex]); + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Moved transform {OldIndex} to position {NewIndex} in pipeline '{Name}'", + index, swapIndex + 1, pipelineName); + + return 0; + } + } + + private static async Task EditSourceAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + string? connection, + string? query, + string? queryFile, + string? massQuery, + string? massQueryFile, + bool dryRun) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + + using (logger.BeginScope(new Dictionary + { + ["Command"] = "pipeline element source edit", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + // Validate query inputs + if (!string.IsNullOrEmpty(query) && !string.IsNullOrEmpty(queryFile)) + { + logger.LogError("Cannot specify both --query and --query-file"); + return 1; + } + + if (!string.IsNullOrEmpty(massQuery) && !string.IsNullOrEmpty(massQueryFile)) + { + logger.LogError("Cannot specify both --mass-query and --mass-query-file"); + return 1; + } + + // Load queries from files if needed + var queryText = query; + if (!string.IsNullOrEmpty(queryFile)) + { + if (!File.Exists(queryFile)) + { + logger.LogError("File not found: {Path}", queryFile); + return 1; + } + queryText = await File.ReadAllTextAsync(queryFile); + } + + var massQueryText = massQuery; + if (!string.IsNullOrEmpty(massQueryFile)) + { + if (!File.Exists(massQueryFile)) + { + logger.LogError("File not found: {Path}", massQueryFile); + return 1; + } + massQueryText = await File.ReadAllTextAsync(massQueryFile); + } + + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + pipeline.Source ??= new SourceElement(); + var changes = new List(); + + if (!string.IsNullOrEmpty(connection)) + { + changes.Add($"Connection: {pipeline.Source.Connection} -> {connection}"); + if (!dryRun) pipeline.Source.Connection = connection; + } + + if (!string.IsNullOrEmpty(queryText)) + { + changes.Add("Query: (updated)"); + if (!dryRun) pipeline.Source.Query = queryText; + } + + if (!string.IsNullOrEmpty(massQueryText)) + { + changes.Add("MassQuery: (updated)"); + if (!dryRun) pipeline.Source.MassQuery = massQueryText; + } + + if (changes.Count == 0) + { + logger.LogWarning("No changes specified"); + return 0; + } + + if (dryRun) + { + logger.LogInformation("[DRY RUN] Would update source:"); + foreach (var change in changes) + { + logger.LogInformation(" {Change}", change); + } + return 0; + } + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Updated source for pipeline '{Name}'", pipelineName); + + return 0; + } + } + + private static async Task EditDestinationAsync( + IServiceProvider serviceProvider, + string? configPath, + bool verbose, + bool quiet, + string pipelineName, + string? table, + string? matchColumns, + string? addMatchColumn, + string? removeMatchColumn, + string? excludeFromUpdate, + string? addExclude, + string? removeExclude, + bool dryRun) + { + var loggerFactory = serviceProvider.GetRequiredService(); + var logger = loggerFactory.CreateLogger("PipelineElementCommands"); + + using (logger.BeginScope(new Dictionary + { + ["Command"] = "pipeline element destination edit", + ["ConfigPath"] = configPath ?? "(default)", + ["PipelineName"] = pipelineName + })) + { + var (pipeline, pipelinePath, folderPath) = await LoadPipelineAsync(serviceProvider, configPath, pipelineName, logger); + if (pipeline == null) + return 1; + + pipeline.Destination ??= new DestinationElement(); + var changes = new List(); + + if (!string.IsNullOrEmpty(table)) + { + changes.Add($"Table: {pipeline.Destination.Table} -> {table}"); + if (!dryRun) pipeline.Destination.Table = table; + } + + // Handle match columns + if (!string.IsNullOrEmpty(matchColumns)) + { + var newColumns = matchColumns.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); + changes.Add($"MatchColumns: [{string.Join(", ", pipeline.Destination.MatchColumns)}] -> [{string.Join(", ", newColumns)}]"); + if (!dryRun) pipeline.Destination.MatchColumns = newColumns; + } + else + { + if (!string.IsNullOrEmpty(addMatchColumn)) + { + if (!pipeline.Destination.MatchColumns.Contains(addMatchColumn, StringComparer.OrdinalIgnoreCase)) + { + changes.Add($"MatchColumns: added '{addMatchColumn}'"); + if (!dryRun) pipeline.Destination.MatchColumns.Add(addMatchColumn); + } + else + { + logger.LogWarning("Match column '{Column}' already exists", addMatchColumn); + } + } + + if (!string.IsNullOrEmpty(removeMatchColumn)) + { + var existingColumn = pipeline.Destination.MatchColumns + .FirstOrDefault(c => c.Equals(removeMatchColumn, StringComparison.OrdinalIgnoreCase)); + if (existingColumn != null) + { + changes.Add($"MatchColumns: removed '{existingColumn}'"); + if (!dryRun) pipeline.Destination.MatchColumns.Remove(existingColumn); + } + else + { + logger.LogWarning("Match column '{Column}' not found", removeMatchColumn); + } + } + } + + // Handle exclude from update + if (!string.IsNullOrEmpty(excludeFromUpdate)) + { + var newExcludes = excludeFromUpdate.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); + changes.Add($"ExcludeFromUpdate: [{string.Join(", ", pipeline.Destination.ExcludeFromUpdate)}] -> [{string.Join(", ", newExcludes)}]"); + if (!dryRun) pipeline.Destination.ExcludeFromUpdate = newExcludes; + } + else + { + if (!string.IsNullOrEmpty(addExclude)) + { + if (!pipeline.Destination.ExcludeFromUpdate.Contains(addExclude, StringComparer.OrdinalIgnoreCase)) + { + changes.Add($"ExcludeFromUpdate: added '{addExclude}'"); + if (!dryRun) pipeline.Destination.ExcludeFromUpdate.Add(addExclude); + } + else + { + logger.LogWarning("Exclude column '{Column}' already exists", addExclude); + } + } + + if (!string.IsNullOrEmpty(removeExclude)) + { + var existingColumn = pipeline.Destination.ExcludeFromUpdate + .FirstOrDefault(c => c.Equals(removeExclude, StringComparison.OrdinalIgnoreCase)); + if (existingColumn != null) + { + changes.Add($"ExcludeFromUpdate: removed '{existingColumn}'"); + if (!dryRun) pipeline.Destination.ExcludeFromUpdate.Remove(existingColumn); + } + else + { + logger.LogWarning("Exclude column '{Column}' not found", removeExclude); + } + } + } + + if (changes.Count == 0) + { + logger.LogWarning("No changes specified"); + return 0; + } + + if (dryRun) + { + logger.LogInformation("[DRY RUN] Would update destination:"); + foreach (var change in changes) + { + logger.LogInformation(" {Change}", change); + } + return 0; + } + + var configFileService = serviceProvider.GetRequiredService(); + await configFileService.SavePipelineAsync(pipelinePath!, pipeline); + + logger.LogInformation("Updated destination for pipeline '{Name}'", pipelineName); + + return 0; + } + } + + #endregion + + #region Helper Methods + + private static async Task<(EtlPipelineConfig? Pipeline, string? Path, string? FolderPath)> LoadPipelineAsync( + IServiceProvider serviceProvider, + string? configPath, + string pipelineName, + ILogger logger) + { + var folderPath = await GetConfigFolderAsync(serviceProvider, configPath); + if (folderPath == null) + { + logger.LogError("Could not find configuration folder. Use --config-path to specify"); + return (null, null, null); + } + + var configFileService = serviceProvider.GetRequiredService(); + var pipelinePath = Path.Combine(folderPath, "Pipelines", $"pipeline.{pipelineName}.json"); + + if (!File.Exists(pipelinePath)) + { + logger.LogError("Pipeline '{Name}' not found", pipelineName); + return (null, null, null); + } + + try + { + var pipeline = await configFileService.LoadPipelineAsync(pipelinePath); + return (pipeline, pipelinePath, folderPath); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to load pipeline"); + return (null, null, null); + } + } + + private static async Task GetConfigFolderAsync(IServiceProvider serviceProvider, string? configPath) + { + if (!string.IsNullOrEmpty(configPath)) + { + if (Directory.Exists(configPath)) + return configPath; + return null; + } + + var autoDiscoveryService = serviceProvider.GetRequiredService(); + return await autoDiscoveryService.FindConfigFolderAsync(); + } + + private static async Task BuildTransformConfigAsync( + string type, + string? configJson, + string? configFile, + string? columns, + string? mappings, + string? dateColumn, + string? timeColumn, + string? outputColumn, + string? column, + string? pattern, + string? replacement, + bool? ignoreCase, + ILogger logger) + { + // If full JSON config is provided, use it + if (!string.IsNullOrEmpty(configJson)) + { + try + { + using var doc = JsonDocument.Parse(configJson); + return doc.RootElement.Clone(); + } + catch (JsonException ex) + { + throw new ArgumentException($"Invalid JSON configuration: {ex.Message}"); + } + } + + // If config file is provided, load it + if (!string.IsNullOrEmpty(configFile)) + { + if (!File.Exists(configFile)) + { + throw new FileNotFoundException($"File not found: {configFile}"); + } + var fileContent = await File.ReadAllTextAsync(configFile); + try + { + using var doc = JsonDocument.Parse(fileContent); + return doc.RootElement.Clone(); + } + catch (JsonException ex) + { + throw new ArgumentException($"Invalid JSON in file: {ex.Message}"); + } + } + + // Build config from type-specific shortcuts + var configDict = new Dictionary(); + + switch (type.ToLowerInvariant()) + { + case "columndrop": + if (!string.IsNullOrEmpty(columns)) + { + configDict["Columns"] = columns.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } + break; + + case "columnrename": + if (!string.IsNullOrEmpty(mappings)) + { + var mappingDict = new Dictionary(); + foreach (var pair in mappings.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var parts = pair.Split('=', 2); + if (parts.Length == 2) + { + mappingDict[parts[0].Trim()] = parts[1].Trim(); + } + } + configDict["Mappings"] = mappingDict; + } + break; + + case "jdedate": + if (!string.IsNullOrEmpty(dateColumn)) + configDict["DateColumn"] = dateColumn; + if (!string.IsNullOrEmpty(timeColumn)) + configDict["TimeColumn"] = timeColumn; + if (!string.IsNullOrEmpty(outputColumn)) + configDict["OutputColumn"] = outputColumn; + break; + + case "regex": + if (!string.IsNullOrEmpty(column)) + configDict["Column"] = column; + if (!string.IsNullOrEmpty(pattern)) + configDict["Pattern"] = pattern; + if (!string.IsNullOrEmpty(replacement)) + configDict["Replacement"] = replacement; + if (ignoreCase.HasValue) + configDict["IgnoreCase"] = ignoreCase.Value; + break; + } + + if (configDict.Count == 0) + { + return null; + } + + var json = JsonSerializer.Serialize(configDict, new JsonSerializerOptions { WriteIndented = false }); + using var document = JsonDocument.Parse(json); + return document.RootElement.Clone(); + } + + #endregion +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Program.cs b/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Program.cs index 10014e9..377fc78 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Program.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager.Cli/Program.cs @@ -81,6 +81,16 @@ public static class Program pipelineCommand.AddCommand(PipelineCommands.CreateDeleteCommand(serviceProvider, configPathOption, verboseOption, quietOption)); pipelineCommand.AddCommand(PipelineCommands.CreateEnableCommand(serviceProvider, configPathOption, verboseOption, quietOption)); pipelineCommand.AddCommand(PipelineCommands.CreateDisableCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + + // Pipeline element subcommand group + var elementCommand = new Command("element", "Manage pipeline elements"); + elementCommand.AddCommand(PipelineElementCommands.CreatePreScriptCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + elementCommand.AddCommand(PipelineElementCommands.CreatePostScriptCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + elementCommand.AddCommand(PipelineElementCommands.CreateTransformCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + elementCommand.AddCommand(PipelineElementCommands.CreateSourceCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + elementCommand.AddCommand(PipelineElementCommands.CreateDestinationCommand(serviceProvider, configPathOption, verboseOption, quietOption)); + pipelineCommand.AddCommand(elementCommand); + rootCommand.AddCommand(pipelineCommand); // Config command group diff --git a/NEW/src/Utils/JdeScoping.ConfigManager.Cli/README.md b/NEW/src/Utils/JdeScoping.ConfigManager.Cli/README.md index bb8edf5..f6c9fd2 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager.Cli/README.md +++ b/NEW/src/Utils/JdeScoping.ConfigManager.Cli/README.md @@ -125,6 +125,161 @@ jdescoping-config pipeline enable jdescoping-config pipeline disable ``` +### Pipeline Element Commands + +Manage elements within a pipeline (pre-scripts, transforms, post-scripts, source, destination). + +#### Pre-Script Management + +```bash +# Add a pre-script +jdescoping-config pipeline element prescript add WorkOrder_Curr \ + --connection lotfinder \ + --script "TRUNCATE TABLE dbo.WorkOrder_Temp;" + +# Add from a SQL file +jdescoping-config pipeline element prescript add WorkOrder_Curr \ + --connection lotfinder \ + --script-file /path/to/script.sql + +# Add at a specific position (1-based index) +jdescoping-config pipeline element prescript add WorkOrder_Curr \ + --script "SELECT 1;" --at-index 1 + +# Remove a pre-script (1-based index) +jdescoping-config pipeline element prescript remove WorkOrder_Curr 2 +jdescoping-config pipeline element prescript remove WorkOrder_Curr 2 --force + +# Edit a pre-script +jdescoping-config pipeline element prescript edit WorkOrder_Curr 1 \ + --script "TRUNCATE TABLE dbo.NewTable;" + +# Move a pre-script up/down +jdescoping-config pipeline element prescript move-up WorkOrder_Curr 3 +jdescoping-config pipeline element prescript move-down WorkOrder_Curr 1 + +# Preview changes without saving +jdescoping-config pipeline element prescript remove WorkOrder_Curr 2 --dry-run +``` + +#### Post-Script Management + +Post-scripts use the same commands as pre-scripts: + +```bash +# Add a post-script +jdescoping-config pipeline element postscript add WorkOrder_Curr \ + --connection lotfinder \ + --script "MERGE INTO dbo.WorkOrder_Hist..." + +# Remove, edit, and move work the same way +jdescoping-config pipeline element postscript remove WorkOrder_Curr 1 --force +jdescoping-config pipeline element postscript edit WorkOrder_Curr 1 --script "..." +jdescoping-config pipeline element postscript move-up WorkOrder_Curr 2 +``` + +#### Transform Management + +```bash +# Add a ColumnDrop transform +jdescoping-config pipeline element transform add WorkOrder_Curr \ + --type ColumnDrop \ + --columns "TempCol1,TempCol2,TempCol3" + +# Add a ColumnRename transform +jdescoping-config pipeline element transform add WorkOrder_Curr \ + --type ColumnRename \ + --mappings "OldName=NewName,AnotherOld=AnotherNew" + +# Add a JdeDate transform +jdescoping-config pipeline element transform add WorkOrder_Curr \ + --type JdeDate \ + --date-column "LastUpdateDate" \ + --time-column "LastUpdateTime" \ + --output-column "LastUpdateDT" + +# Add a Regex transform +jdescoping-config pipeline element transform add WorkOrder_Curr \ + --type Regex \ + --column "Description" \ + --pattern "^\s+" \ + --replacement "" \ + --ignore-case + +# Add with full JSON config +jdescoping-config pipeline element transform add WorkOrder_Curr \ + --type ColumnDrop \ + --config '{"Columns":["Col1","Col2"]}' + +# Add from config file +jdescoping-config pipeline element transform add WorkOrder_Curr \ + --type JdeDate \ + --config-file /path/to/transform-config.json + +# Remove a transform (1-based index) +jdescoping-config pipeline element transform remove WorkOrder_Curr 2 + +# Edit a transform +jdescoping-config pipeline element transform edit WorkOrder_Curr 1 \ + --columns "NewCol1,NewCol2" + +# Move transforms +jdescoping-config pipeline element transform move-up WorkOrder_Curr 3 +jdescoping-config pipeline element transform move-down WorkOrder_Curr 1 +``` + +#### Source Configuration + +```bash +# Edit source connection +jdescoping-config pipeline element source edit WorkOrder_Curr \ + --connection jde + +# Edit source query +jdescoping-config pipeline element source edit WorkOrder_Curr \ + --query "SELECT * FROM F4801 WHERE WADOCO > :lastSync" + +# Edit from file +jdescoping-config pipeline element source edit WorkOrder_Curr \ + --query-file /path/to/query.sql + +# Edit mass sync query +jdescoping-config pipeline element source edit WorkOrder_Curr \ + --mass-query "SELECT * FROM F4801" + +# Edit multiple settings +jdescoping-config pipeline element source edit WorkOrder_Curr \ + --connection jde \ + --query-file /path/to/incremental.sql \ + --mass-query-file /path/to/mass.sql +``` + +#### Destination Configuration + +```bash +# Edit destination table +jdescoping-config pipeline element destination edit WorkOrder_Curr \ + --table "dbo.WorkOrder_Curr" + +# Replace all match columns +jdescoping-config pipeline element destination edit WorkOrder_Curr \ + --match-columns "OrderNumber,LineNumber" + +# Add/remove individual match columns +jdescoping-config pipeline element destination edit WorkOrder_Curr \ + --add-match-column "NewColumn" +jdescoping-config pipeline element destination edit WorkOrder_Curr \ + --remove-match-column "OldColumn" + +# Manage exclude-from-update columns +jdescoping-config pipeline element destination edit WorkOrder_Curr \ + --exclude-from-update "CreatedDate,CreatedBy" +jdescoping-config pipeline element destination edit WorkOrder_Curr \ + --add-exclude "AuditColumn" +jdescoping-config pipeline element destination edit WorkOrder_Curr \ + --remove-exclude "OldAuditColumn" +``` + ### Configuration Viewing Commands View configuration settings (read-only). diff --git a/NEW/tests/Utils/JdeScoping.ConfigManager.Cli.Tests/Commands/PipelineElementCommandsTests.cs b/NEW/tests/Utils/JdeScoping.ConfigManager.Cli.Tests/Commands/PipelineElementCommandsTests.cs new file mode 100644 index 0000000..f64ad06 --- /dev/null +++ b/NEW/tests/Utils/JdeScoping.ConfigManager.Cli.Tests/Commands/PipelineElementCommandsTests.cs @@ -0,0 +1,983 @@ +using System.CommandLine; +using System.Text.Json; +using JdeScoping.ConfigManager.Cli.Commands; +using JdeScoping.ConfigManager.Core.Services; +using JdeScoping.DataSync.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace JdeScoping.ConfigManager.Cli.Tests.Commands; + +[Collection("Console Tests")] +public class PipelineElementCommandsTests +{ + private readonly IServiceProvider _serviceProvider; + private readonly IConfigFileService _configFileService; + private readonly IAutoDiscoveryService _autoDiscoveryService; + private readonly Option _configPathOption; + private readonly Option _verboseOption; + private readonly Option _quietOption; + + public PipelineElementCommandsTests() + { + _configFileService = Substitute.For(); + _autoDiscoveryService = Substitute.For(); + + var services = new ServiceCollection(); + services.AddSingleton(_configFileService); + services.AddSingleton(_autoDiscoveryService); + services.AddTestLogging(); + _serviceProvider = services.BuildServiceProvider(); + + _configPathOption = new Option(["--config-path", "-c"]); + _verboseOption = new Option(["--verbose", "-v"]); + _quietOption = new Option(["--quiet", "-q"]); + } + + #region Command Structure Tests + + [Fact] + public void CreatePreScriptCommand_ReturnsCommandWithSubcommands() + { + // Act + var command = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + + // Assert + command.ShouldNotBeNull(); + command.Name.ShouldBe("prescript"); + command.Subcommands.ShouldContain(c => c.Name == "add"); + command.Subcommands.ShouldContain(c => c.Name == "remove"); + command.Subcommands.ShouldContain(c => c.Name == "edit"); + command.Subcommands.ShouldContain(c => c.Name == "move-up"); + command.Subcommands.ShouldContain(c => c.Name == "move-down"); + } + + [Fact] + public void CreatePostScriptCommand_ReturnsCommandWithSubcommands() + { + // Act + var command = PipelineElementCommands.CreatePostScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + + // Assert + command.ShouldNotBeNull(); + command.Name.ShouldBe("postscript"); + command.Subcommands.ShouldContain(c => c.Name == "add"); + command.Subcommands.ShouldContain(c => c.Name == "remove"); + command.Subcommands.ShouldContain(c => c.Name == "edit"); + command.Subcommands.ShouldContain(c => c.Name == "move-up"); + command.Subcommands.ShouldContain(c => c.Name == "move-down"); + } + + [Fact] + public void CreateTransformCommand_ReturnsCommandWithSubcommands() + { + // Act + var command = PipelineElementCommands.CreateTransformCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + + // Assert + command.ShouldNotBeNull(); + command.Name.ShouldBe("transform"); + command.Subcommands.ShouldContain(c => c.Name == "add"); + command.Subcommands.ShouldContain(c => c.Name == "remove"); + command.Subcommands.ShouldContain(c => c.Name == "edit"); + command.Subcommands.ShouldContain(c => c.Name == "move-up"); + command.Subcommands.ShouldContain(c => c.Name == "move-down"); + } + + [Fact] + public void CreateSourceCommand_ReturnsCommandWithEditSubcommand() + { + // Act + var command = PipelineElementCommands.CreateSourceCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + + // Assert + command.ShouldNotBeNull(); + command.Name.ShouldBe("source"); + command.Subcommands.ShouldContain(c => c.Name == "edit"); + } + + [Fact] + public void CreateDestinationCommand_ReturnsCommandWithEditSubcommand() + { + // Act + var command = PipelineElementCommands.CreateDestinationCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + + // Assert + command.ShouldNotBeNull(); + command.Name.ShouldBe("destination"); + command.Subcommands.ShouldContain(c => c.Name == "edit"); + } + + [Fact] + public void PreScriptAddCommand_HasRequiredOptions() + { + // Act + var prescript = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var addCommand = prescript.Subcommands.First(c => c.Name == "add"); + + // Assert + addCommand.Arguments.ShouldContain(a => a.Name == "pipeline-name"); + addCommand.Options.ShouldContain(o => o.Name == "connection"); + addCommand.Options.ShouldContain(o => o.Name == "script"); + addCommand.Options.ShouldContain(o => o.Name == "script-file"); + addCommand.Options.ShouldContain(o => o.Name == "at-index"); + } + + [Fact] + public void TransformAddCommand_HasRequiredOptions() + { + // Act + var transform = PipelineElementCommands.CreateTransformCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var addCommand = transform.Subcommands.First(c => c.Name == "add"); + + // Assert + addCommand.Arguments.ShouldContain(a => a.Name == "pipeline-name"); + addCommand.Options.ShouldContain(o => o.Name == "type"); + addCommand.Options.ShouldContain(o => o.Name == "config"); + addCommand.Options.ShouldContain(o => o.Name == "columns"); + addCommand.Options.ShouldContain(o => o.Name == "mappings"); + addCommand.Options.ShouldContain(o => o.Name == "date-column"); + addCommand.Options.ShouldContain(o => o.Name == "time-column"); + addCommand.Options.ShouldContain(o => o.Name == "output-column"); + addCommand.Options.ShouldContain(o => o.Name == "column"); + addCommand.Options.ShouldContain(o => o.Name == "pattern"); + addCommand.Options.ShouldContain(o => o.Name == "replacement"); + addCommand.Options.ShouldContain(o => o.Name == "ignore-case"); + } + + [Fact] + public void SourceEditCommand_HasRequiredOptions() + { + // Act + var source = PipelineElementCommands.CreateSourceCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var editCommand = source.Subcommands.First(c => c.Name == "edit"); + + // Assert + editCommand.Arguments.ShouldContain(a => a.Name == "pipeline-name"); + editCommand.Options.ShouldContain(o => o.Name == "connection"); + editCommand.Options.ShouldContain(o => o.Name == "query"); + editCommand.Options.ShouldContain(o => o.Name == "query-file"); + editCommand.Options.ShouldContain(o => o.Name == "mass-query"); + editCommand.Options.ShouldContain(o => o.Name == "mass-query-file"); + editCommand.Options.ShouldContain(o => o.Name == "dry-run"); + } + + [Fact] + public void DestinationEditCommand_HasRequiredOptions() + { + // Act + var destination = PipelineElementCommands.CreateDestinationCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var editCommand = destination.Subcommands.First(c => c.Name == "edit"); + + // Assert + editCommand.Arguments.ShouldContain(a => a.Name == "pipeline-name"); + editCommand.Options.ShouldContain(o => o.Name == "table"); + editCommand.Options.ShouldContain(o => o.Name == "match-columns"); + editCommand.Options.ShouldContain(o => o.Name == "add-match-column"); + editCommand.Options.ShouldContain(o => o.Name == "remove-match-column"); + editCommand.Options.ShouldContain(o => o.Name == "exclude-from-update"); + editCommand.Options.ShouldContain(o => o.Name == "add-exclude"); + editCommand.Options.ShouldContain(o => o.Name == "remove-exclude"); + editCommand.Options.ShouldContain(o => o.Name == "dry-run"); + } + + #endregion + + #region PreScript Add Tests + + [Fact] + public async Task PreScriptAddCommand_AddsScriptToPipeline() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); // File must exist for the command + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + PreScripts = [] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var prescript = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { prescript }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act + await rootCommand.InvokeAsync(["prescript", "add", "TestPipeline", "--script", "SELECT 1;"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.PreScripts.Count.ShouldBe(1); + savedPipeline.PreScripts[0].Script.ShouldBe("SELECT 1;"); + savedPipeline.PreScripts[0].Connection.ShouldBe("lotfinder"); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task PreScriptAddCommand_InsertsAtSpecifiedIndex() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + PreScripts = + [ + new ScriptElement { Script = "Script1" }, + new ScriptElement { Script = "Script2" } + ] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var prescript = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { prescript }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act - insert at position 2 (between Script1 and Script2) + await rootCommand.InvokeAsync(["prescript", "add", "TestPipeline", "--script", "NewScript", "--at-index", "2"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.PreScripts.Count.ShouldBe(3); + savedPipeline.PreScripts[0].Script.ShouldBe("Script1"); + savedPipeline.PreScripts[1].Script.ShouldBe("NewScript"); + savedPipeline.PreScripts[2].Script.ShouldBe("Script2"); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + #endregion + + #region PreScript Remove Tests + + [Fact] + public async Task PreScriptRemoveCommand_RemovesScriptWithForce() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + PreScripts = + [ + new ScriptElement { Script = "Script1" }, + new ScriptElement { Script = "Script2" }, + new ScriptElement { Script = "Script3" } + ] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var prescript = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { prescript }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act - remove script at index 2 (Script2) + await rootCommand.InvokeAsync(["prescript", "remove", "TestPipeline", "2", "--force"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.PreScripts.Count.ShouldBe(2); + savedPipeline.PreScripts[0].Script.ShouldBe("Script1"); + savedPipeline.PreScripts[1].Script.ShouldBe("Script3"); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task PreScriptRemoveCommand_DryRunDoesNotSave() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + PreScripts = [new ScriptElement { Script = "Script1" }] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + var prescript = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { prescript }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act + await rootCommand.InvokeAsync(["prescript", "remove", "TestPipeline", "1", "--dry-run"]); + + // Assert - SavePipelineAsync should not be called + await _configFileService.DidNotReceive() + .SavePipelineAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + #endregion + + #region PreScript Move Tests + + [Fact] + public async Task PreScriptMoveUpCommand_SwapsWithPreviousElement() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + PreScripts = + [ + new ScriptElement { Script = "Script1" }, + new ScriptElement { Script = "Script2" }, + new ScriptElement { Script = "Script3" } + ] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var prescript = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { prescript }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act - move Script2 (index 2) up + await rootCommand.InvokeAsync(["prescript", "move-up", "TestPipeline", "2"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.PreScripts[0].Script.ShouldBe("Script2"); + savedPipeline.PreScripts[1].Script.ShouldBe("Script1"); + savedPipeline.PreScripts[2].Script.ShouldBe("Script3"); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task PreScriptMoveDownCommand_SwapsWithNextElement() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + PreScripts = + [ + new ScriptElement { Script = "Script1" }, + new ScriptElement { Script = "Script2" }, + new ScriptElement { Script = "Script3" } + ] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var prescript = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { prescript }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act - move Script2 (index 2) down + await rootCommand.InvokeAsync(["prescript", "move-down", "TestPipeline", "2"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.PreScripts[0].Script.ShouldBe("Script1"); + savedPipeline.PreScripts[1].Script.ShouldBe("Script3"); + savedPipeline.PreScripts[2].Script.ShouldBe("Script2"); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + #endregion + + #region Transform Add Tests + + [Fact] + public async Task TransformAddCommand_AddsColumnDropTransform() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + Transforms = [] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var transform = PipelineElementCommands.CreateTransformCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { transform }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act + await rootCommand.InvokeAsync(["transform", "add", "TestPipeline", "--type", "ColumnDrop", "--columns", "Col1,Col2,Col3"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.Transforms.Count.ShouldBe(1); + savedPipeline.Transforms[0].TransformType.ShouldBe("ColumnDrop"); + savedPipeline.Transforms[0].Config.ShouldNotBeNull(); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task TransformAddCommand_AddsJdeDateTransform() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + Transforms = [] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var transform = PipelineElementCommands.CreateTransformCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { transform }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act + await rootCommand.InvokeAsync(["transform", "add", "TestPipeline", + "--type", "JdeDate", + "--date-column", "WADOCO", + "--time-column", "WATIME", + "--output-column", "UpdatedDT"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.Transforms.Count.ShouldBe(1); + savedPipeline.Transforms[0].TransformType.ShouldBe("JdeDate"); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + #endregion + + #region Source Edit Tests + + [Fact] + public async Task SourceEditCommand_UpdatesConnection() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + Source = new SourceElement { Connection = "jde", Query = "SELECT 1" } + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var source = PipelineElementCommands.CreateSourceCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { source }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act + await rootCommand.InvokeAsync(["source", "edit", "TestPipeline", "--connection", "cms"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.Source.Connection.ShouldBe("cms"); + savedPipeline.Source.Query.ShouldBe("SELECT 1"); // Unchanged + } + finally + { + Directory.Delete(tempDir, true); + } + } + + #endregion + + #region Destination Edit Tests + + [Fact] + public async Task DestinationEditCommand_UpdatesMatchColumns() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + Destination = new DestinationElement + { + Table = "dbo.TestTable", + MatchColumns = ["OldCol1", "OldCol2"] + } + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var destination = PipelineElementCommands.CreateDestinationCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { destination }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act + await rootCommand.InvokeAsync(["destination", "edit", "TestPipeline", "--match-columns", "NewCol1,NewCol2,NewCol3"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.Destination.MatchColumns.Count.ShouldBe(3); + savedPipeline.Destination.MatchColumns.ShouldContain("NewCol1"); + savedPipeline.Destination.MatchColumns.ShouldContain("NewCol2"); + savedPipeline.Destination.MatchColumns.ShouldContain("NewCol3"); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task DestinationEditCommand_AddsMatchColumn() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + EtlPipelineConfig? savedPipeline = null; + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + Destination = new DestinationElement + { + Table = "dbo.TestTable", + MatchColumns = ["Col1", "Col2"] + } + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + _configFileService.SavePipelineAsync(pipelinePath, Arg.Any(), Arg.Any()) + .Returns(ci => + { + savedPipeline = ci.ArgAt(1); + return Task.CompletedTask; + }); + + var destination = PipelineElementCommands.CreateDestinationCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { destination }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act + await rootCommand.InvokeAsync(["destination", "edit", "TestPipeline", "--add-match-column", "Col3"]); + + // Assert + savedPipeline.ShouldNotBeNull(); + savedPipeline.Destination.MatchColumns.Count.ShouldBe(3); + savedPipeline.Destination.MatchColumns.ShouldContain("Col1"); + savedPipeline.Destination.MatchColumns.ShouldContain("Col2"); + savedPipeline.Destination.MatchColumns.ShouldContain("Col3"); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + #endregion + + #region Error Handling Tests + + [Fact] + public async Task PreScriptRemoveCommand_InvalidIndex_ReturnsError() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + PreScripts = [new ScriptElement { Script = "Script1" }] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + var prescript = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { prescript }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act - try to remove index 5 when only 1 exists + await rootCommand.InvokeAsync(["prescript", "remove", "TestPipeline", "5", "--force"]); + + // Assert - SavePipelineAsync should not be called + await _configFileService.DidNotReceive() + .SavePipelineAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task TransformAddCommand_InvalidType_ReturnsError() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + Transforms = [] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + var transform = PipelineElementCommands.CreateTransformCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { transform }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act - try to add invalid transform type + await rootCommand.InvokeAsync(["transform", "add", "TestPipeline", "--type", "InvalidType"]); + + // Assert - SavePipelineAsync should not be called + await _configFileService.DidNotReceive() + .SavePipelineAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task PreScriptMoveUpCommand_AtFirstPosition_DoesNotSave() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var pipelinesDir = Path.Combine(tempDir, "Pipelines"); + Directory.CreateDirectory(pipelinesDir); + var pipelinePath = Path.Combine(pipelinesDir, "pipeline.TestPipeline.json"); + File.WriteAllText(pipelinePath, "{}"); + + try + { + _autoDiscoveryService.FindConfigFolderAsync(Arg.Any()) + .Returns(Task.FromResult(tempDir)); + + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + PreScripts = + [ + new ScriptElement { Script = "Script1" }, + new ScriptElement { Script = "Script2" } + ] + }; + + _configFileService.LoadPipelineAsync(pipelinePath, Arg.Any()) + .Returns(Task.FromResult(pipeline)); + + var prescript = PipelineElementCommands.CreatePreScriptCommand( + _serviceProvider, _configPathOption, _verboseOption, _quietOption); + var rootCommand = new RootCommand { prescript }; + rootCommand.AddGlobalOption(_configPathOption); + rootCommand.AddGlobalOption(_verboseOption); + rootCommand.AddGlobalOption(_quietOption); + + // Act - try to move first element up (should be no-op) + await rootCommand.InvokeAsync(["prescript", "move-up", "TestPipeline", "1"]); + + // Assert - SavePipelineAsync should not be called (already at first position) + await _configFileService.DidNotReceive() + .SavePipelineAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + #endregion +}