feat: add startup config validation and document ConfigManager pipeline editor

Add ConfigurationValidationRunner with IConfigurationValidator interface for
validating required settings at startup. Includes SecureStore and LDAP validators.
Expand ConfigManager with pipeline editing UI, dialogs, and step editors.
Update documentation with config validation guidance.
This commit is contained in:
Joseph Doherty
2026-01-21 17:47:15 -05:00
parent ceb63bfefb
commit e5fe2f06e9
88 changed files with 4995 additions and 201 deletions
@@ -59,11 +59,21 @@ public class PipelineModel
/// </summary>
public PipelineSchedules Schedules { get; set; } = new();
/// <summary>
/// Gets or sets optional data transformers applied between source and destination.
/// </summary>
public List<TransformerModel>? Transformers { get; set; }
/// <summary>
/// Gets or sets the destination configuration for data loading.
/// </summary>
public PipelineDestination Destination { get; set; } = new();
/// <summary>
/// Gets or sets optional scripts to execute before pipeline starts.
/// </summary>
public string[]? PreScripts { get; set; }
/// <summary>
/// Gets or sets optional scripts to execute after pipeline completion.
/// </summary>
@@ -74,23 +84,33 @@ public class PipelineSource
{
/// <summary>
/// Gets or sets the source database connection name.
/// Used for database sources.
/// </summary>
public string Connection { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the query to extract data from the source.
/// Used for database sources.
/// </summary>
public string Query { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the optional mass query for full data extraction.
/// Used for database sources.
/// </summary>
public string? MassQuery { get; set; }
/// <summary>
/// Gets or sets the query parameters and their definitions.
/// Used for database sources.
/// </summary>
public Dictionary<string, ParameterDefinition> Parameters { get; set; } = new();
/// <summary>
/// Gets or sets the file name for file-based sources.
/// Used for Protobuf+Zstd files.
/// </summary>
public string? FileName { get; set; }
}
public class ParameterDefinition
@@ -131,6 +151,12 @@ public class PipelineSchedules
public class PipelineDestination
{
/// <summary>
/// Gets or sets the destination type (BulkImport or BulkMerge).
/// BulkImport truncates and loads; BulkMerge matches and updates.
/// </summary>
public string Type { get; set; } = "BulkMerge";
/// <summary>
/// Gets or sets the destination table name.
/// </summary>
@@ -138,11 +164,52 @@ public class PipelineDestination
/// <summary>
/// Gets or sets the columns used to match existing records for updates.
/// Only used for BulkMerge destination type.
/// </summary>
public string[] MatchColumns { get; set; } = [];
/// <summary>
/// Gets or sets the columns to exclude from update operations.
/// Only used for BulkMerge destination type.
/// </summary>
public string[] ExcludeFromUpdate { get; set; } = [];
}
/// <summary>
/// Represents a data transformer applied between source and destination.
/// </summary>
public class TransformerModel
{
/// <summary>
/// Gets or sets the transformer type.
/// Supported types: ColumnDrop, ColumnRename, JdeDate.
/// </summary>
public string Type { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the columns affected by this transformer.
/// Used by ColumnDrop (columns to remove) and JdeDate (date/time columns).
/// </summary>
public List<string>? Columns { get; set; }
/// <summary>
/// Gets or sets the column mappings for rename operations.
/// Used by ColumnRename: OldName → NewName.
/// </summary>
public Dictionary<string, string>? Mappings { get; set; }
/// <summary>
/// Gets or sets the date column name for JdeDate transformer.
/// </summary>
public string? DateColumn { get; set; }
/// <summary>
/// Gets or sets the time column name for JdeDate transformer.
/// </summary>
public string? TimeColumn { get; set; }
/// <summary>
/// Gets or sets the output column name for JdeDate transformer.
/// </summary>
public string? OutputColumn { get; set; }
}