refactor(configmanager): migrate to per-file pipeline system

Align ConfigManager with DataSync's per-file pipeline format (pipeline.*.json)
by reusing EtlPipelineConfig types directly, eliminating duplicate models and
simplifying the codebase. Removes ~3200 lines of obsolete code.
This commit is contained in:
Joseph Doherty
2026-01-23 02:30:48 -05:00
parent 1b7bb26def
commit ba54a87be5
49 changed files with 1429 additions and 4396 deletions
@@ -17,7 +17,7 @@
<PackageReference Include="Avalonia.Fonts.Inter" Version="11.2.*" />
<PackageReference Include="Avalonia.Diagnostics" Version="11.2.*" Condition="'$(Configuration)' == 'Debug'" />
<PackageReference Include="DiffPlex" Version="1.7.*" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.0.*" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.1.*" />
<PackageReference Include="MessageBox.Avalonia" Version="3.1.*" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.*" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.*" />
@@ -31,6 +31,7 @@
<ItemGroup>
<ProjectReference Include="..\..\JdeScoping.Core\JdeScoping.Core.csproj" />
<ProjectReference Include="..\..\JdeScoping.DataSync\JdeScoping.DataSync.csproj" />
<ProjectReference Include="..\..\JdeScoping.Infrastructure\JdeScoping.Infrastructure.csproj" />
</ItemGroup>
</Project>
@@ -273,7 +273,7 @@ public class SecureStoreSection
public class PipelinesSection
{
/// <summary>
/// Gets or sets the path to the pipelines configuration file.
/// Gets or sets the directory containing pipeline.*.json files.
/// </summary>
public string ConfigPath { get; set; } = "Pipelines/pipelines.json";
public string ConfigDirectory { get; set; } = "Pipelines";
}
@@ -1,256 +0,0 @@
using System.Text.Json.Serialization;
namespace JdeScoping.ConfigManager.Models;
/// <summary>
/// Root model for pipelines.json configuration.
/// </summary>
public class PipelinesConfigModel
{
/// <summary>
/// Gets or sets the pipeline settings.
/// </summary>
public PipelineSettings Settings { get; set; } = new();
/// <summary>
/// Gets or sets the default schedules for all pipelines.
/// </summary>
public ScheduleDefaults ScheduleDefaults { get; set; } = new();
/// <summary>
/// Gets or sets the collection of named pipelines.
/// </summary>
public Dictionary<string, PipelineModel> Pipelines { get; set; } = new();
}
public class PipelineSettings
{
/// <summary>
/// Gets or sets the timezone for scheduling operations.
/// </summary>
public string Timezone { get; set; } = "UTC";
}
public class ScheduleDefaults
{
/// <summary>
/// Gets or sets the default mass data refresh schedule.
/// </summary>
public ScheduleModel Mass { get; set; } = new() { Enabled = true, IntervalMinutes = 10080, PrePurge = true, ReIndex = true };
/// <summary>
/// Gets or sets the default daily data refresh schedule.
/// </summary>
public ScheduleModel Daily { get; set; } = new() { Enabled = true, IntervalMinutes = 1440 };
/// <summary>
/// Gets or sets the default hourly data refresh schedule.
/// </summary>
public ScheduleModel Hourly { get; set; } = new() { Enabled = true, IntervalMinutes = 60 };
}
public class PipelineModel
{
/// <summary>
/// Gets or sets the source configuration for data extraction.
/// </summary>
public PipelineSource Source { get; set; } = new();
/// <summary>
/// Gets or sets the schedule configurations for this pipeline.
/// </summary>
public PipelineSchedules Schedules { get; set; } = new();
/// <summary>
/// Gets or sets optional data transformers applied between source and destination.
/// </summary>
public List<TransformerModel>? Transformers { get; set; }
/// <summary>
/// Gets or sets the destination configuration for data loading.
/// </summary>
public PipelineDestination Destination { get; set; } = new();
/// <summary>
/// Gets or sets optional scripts to execute before pipeline starts.
/// </summary>
public string[]? PreScripts { get; set; }
/// <summary>
/// Gets or sets optional scripts to execute after pipeline completion.
/// </summary>
public string[]? PostScripts { get; set; }
}
public class PipelineSource
{
/// <summary>
/// Gets or sets the source database connection name.
/// Used for database sources.
/// </summary>
public string Connection { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the query to extract data from the source.
/// Used for database sources.
/// </summary>
public string Query { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the optional mass query for full data extraction.
/// Used for database sources.
/// </summary>
public string? MassQuery { get; set; }
/// <summary>
/// Gets or sets the query parameters and their definitions.
/// Used for database sources.
/// </summary>
public Dictionary<string, ParameterDefinition> Parameters { get; set; } = new();
/// <summary>
/// Gets or sets the file name for file-based sources.
/// Used for Protobuf+Zstd files.
/// </summary>
public string? FileName { get; set; }
}
public class ParameterDefinition
{
/// <summary>
/// Gets or sets the parameter name.
/// </summary>
public string Name { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the optional parameter format string.
/// </summary>
public string? Format { get; set; }
/// <summary>
/// Gets or sets the optional parameter source or derivation logic.
/// </summary>
public string? Source { get; set; }
}
public class PipelineSchedules
{
/// <summary>
/// Gets or sets the mass refresh schedule for this pipeline.
/// </summary>
public ScheduleModel? Mass { get; set; }
/// <summary>
/// Gets or sets the daily refresh schedule for this pipeline.
/// </summary>
public ScheduleModel? Daily { get; set; }
/// <summary>
/// Gets or sets the hourly refresh schedule for this pipeline.
/// </summary>
public ScheduleModel? Hourly { get; set; }
}
public class PipelineDestination
{
/// <summary>
/// Gets or sets the destination type (BulkImport or BulkMerge).
/// BulkImport truncates and loads; BulkMerge matches and updates.
/// </summary>
public string Type { get; set; } = "BulkMerge";
/// <summary>
/// Gets or sets the destination table name.
/// </summary>
public string Table { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the columns used to match existing records for updates.
/// Only used for BulkMerge destination type.
/// </summary>
public string[] MatchColumns { get; set; } = [];
/// <summary>
/// Gets or sets the columns to exclude from update operations.
/// Only used for BulkMerge destination type.
/// </summary>
public string[] ExcludeFromUpdate { get; set; } = [];
}
/// <summary>
/// Represents a data transformer applied between source and destination.
/// </summary>
public class TransformerModel
{
/// <summary>
/// Gets or sets the transformer type.
/// Supported types: ColumnDrop, ColumnRename, JdeDate.
/// </summary>
public string Type { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the columns affected by this transformer.
/// Used by ColumnDrop (columns to remove) and JdeDate (date/time columns).
/// </summary>
public List<string>? Columns { get; set; }
/// <summary>
/// Gets or sets the column mappings for rename operations.
/// Used by ColumnRename: OldName → NewName.
/// </summary>
public Dictionary<string, string>? Mappings { get; set; }
/// <summary>
/// Gets or sets the date column name for JdeDate transformer.
/// </summary>
public string? DateColumn { get; set; }
/// <summary>
/// Gets or sets the time column name for JdeDate transformer.
/// </summary>
public string? TimeColumn { get; set; }
/// <summary>
/// Gets or sets the output column name for JdeDate transformer.
/// </summary>
public string? OutputColumn { get; set; }
/// <summary>
/// Gets or sets the column name for Regex transformer.
/// </summary>
public string? ColumnName { get; set; }
/// <summary>
/// Gets or sets the regex pattern for Regex transformer.
/// </summary>
public string? Pattern { get; set; }
/// <summary>
/// Gets or sets the replacement string for Regex transformer (null = Match &amp; Extract mode).
/// </summary>
public string? Replacement { get; set; }
/// <summary>
/// Gets or sets whether regex matching is case-insensitive.
/// </summary>
public bool IgnoreCase { get; set; }
/// <summary>
/// Gets or sets the behavior when regex pattern does not match.
/// </summary>
public NonMatchBehavior NonMatchBehavior { get; set; } = NonMatchBehavior.KeepOriginal;
}
/// <summary>
/// Specifies behavior when a regex pattern does not match the input value.
/// </summary>
[JsonConverter(typeof(JsonStringEnumConverter))]
public enum NonMatchBehavior
{
/// <summary>Keep the original value unchanged.</summary>
KeepOriginal,
/// <summary>Return null/DBNull.</summary>
ReturnNull,
/// <summary>Return an empty string.</summary>
ReturnEmpty
}
@@ -1,27 +0,0 @@
namespace JdeScoping.ConfigManager.Models;
/// <summary>
/// Model for schedule configuration.
/// </summary>
public class ScheduleModel
{
/// <summary>
/// Gets or sets a value indicating whether the scheduled task is enabled.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Gets or sets the interval in minutes between scheduled task executions.
/// </summary>
public int IntervalMinutes { get; set; } = 60;
/// <summary>
/// Gets or sets a value indicating whether to purge data before task execution.
/// </summary>
public bool PrePurge { get; set; } = false;
/// <summary>
/// Gets or sets a value indicating whether to reindex after task execution.
/// </summary>
public bool ReIndex { get; set; } = false;
}
@@ -1,6 +1,7 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using JdeScoping.ConfigManager.Models;
using JdeScoping.DataSync.Configuration;
using Microsoft.Extensions.Logging;
namespace JdeScoping.ConfigManager.Services;
@@ -55,29 +56,6 @@ public class ConfigFileService : IConfigFileService
}
}
/// <summary>
/// Loads the pipelines configuration from the specified file path.
/// </summary>
/// <param name="path">The file path to load pipelines from.</param>
/// <param name="ct">Cancellation token for the operation.</param>
/// <returns>The loaded pipelines configuration model or a new empty model if deserialization fails.</returns>
/// <exception cref="ConfigLoadException">Thrown when the JSON cannot be parsed.</exception>
public async Task<PipelinesConfigModel> LoadPipelinesAsync(string path, CancellationToken ct = default)
{
_logger?.LogInformation("Loading pipelines from {Path}", path);
try
{
var json = await _fileSystem.ReadAllTextAsync(path, ct);
var config = JsonSerializer.Deserialize<PipelinesConfigModel>(json, JsonOptions);
return config ?? new PipelinesConfigModel();
}
catch (JsonException ex)
{
throw new ConfigLoadException(path, $"Failed to parse pipelines.json: {ex.Message}", ex);
}
}
/// <summary>
/// Saves the application settings configuration to the specified file path.
/// </summary>
@@ -92,15 +70,100 @@ public class ConfigFileService : IConfigFileService
}
/// <summary>
/// Saves the pipelines configuration to the specified file path.
/// Loads a single pipeline from a pipeline.*.json file.
/// </summary>
/// <param name="path">The file path to save pipelines to.</param>
/// <param name="config">The pipelines configuration model to save.</param>
/// <param name="path">The file path to the pipeline file.</param>
/// <param name="ct">Cancellation token for the operation.</param>
public async Task SavePipelinesAsync(string path, PipelinesConfigModel config, CancellationToken ct = default)
/// <returns>The loaded pipeline configuration.</returns>
/// <exception cref="ConfigLoadException">Thrown when the JSON cannot be parsed.</exception>
public async Task<EtlPipelineConfig> LoadPipelineAsync(string path, CancellationToken ct = default)
{
_logger?.LogInformation("Saving pipelines to {Path}", path);
_logger?.LogDebug("Loading pipeline from {Path}", path);
try
{
var json = await _fileSystem.ReadAllTextAsync(path, ct);
var config = JsonSerializer.Deserialize<EtlPipelineConfig>(json, JsonOptions);
return config ?? throw new ConfigLoadException(path, "Pipeline file was empty or invalid.");
}
catch (JsonException ex)
{
throw new ConfigLoadException(path, $"Failed to parse pipeline file: {ex.Message}", ex);
}
}
/// <summary>
/// Saves a single pipeline to a pipeline.*.json file.
/// </summary>
/// <param name="path">The file path to save the pipeline to.</param>
/// <param name="config">The pipeline configuration to save.</param>
/// <param name="ct">Cancellation token for the operation.</param>
public async Task SavePipelineAsync(string path, EtlPipelineConfig config, CancellationToken ct = default)
{
_logger?.LogDebug("Saving pipeline to {Path}", path);
var json = JsonSerializer.Serialize(config, JsonOptions);
await _fileSystem.WriteAllTextAsync(path, json, ct);
}
/// <summary>
/// Loads all pipelines from a directory containing pipeline.*.json files.
/// </summary>
/// <param name="directory">The directory containing pipeline files.</param>
/// <param name="ct">Cancellation token for the operation.</param>
/// <returns>A dictionary of pipeline name to configuration.</returns>
public async Task<Dictionary<string, EtlPipelineConfig>> LoadAllPipelinesAsync(string directory, CancellationToken ct = default)
{
_logger?.LogInformation("Loading all pipelines from {Directory}", directory);
var pipelines = new Dictionary<string, EtlPipelineConfig>(StringComparer.OrdinalIgnoreCase);
if (!_fileSystem.DirectoryExists(directory))
{
_logger?.LogWarning("Pipeline directory does not exist: {Directory}", directory);
return pipelines;
}
var files = await _fileSystem.GetFilesAsync(directory, "pipeline.*.json", ct);
foreach (var file in files)
{
try
{
var pipeline = await LoadPipelineAsync(file, ct);
var fileName = _fileSystem.GetFileNameWithoutExtension(file);
// Extract name from "pipeline.{name}" format
var name = fileName.StartsWith("pipeline.", StringComparison.OrdinalIgnoreCase)
? fileName["pipeline.".Length..]
: fileName;
// Ensure the pipeline name matches the filename
if (string.IsNullOrEmpty(pipeline.Name))
{
pipeline.Name = name;
}
pipelines[name] = pipeline;
_logger?.LogDebug("Loaded pipeline: {Name}", name);
}
catch (ConfigLoadException ex)
{
_logger?.LogWarning(ex, "Failed to load pipeline from {File}", file);
}
}
_logger?.LogInformation("Loaded {Count} pipelines from {Directory}", pipelines.Count, directory);
return pipelines;
}
/// <summary>
/// Deletes a pipeline file.
/// </summary>
/// <param name="path">The file path to delete.</param>
/// <param name="ct">Cancellation token for the operation.</param>
public async Task DeletePipelineFileAsync(string path, CancellationToken ct = default)
{
_logger?.LogInformation("Deleting pipeline file: {Path}", path);
await _fileSystem.DeleteFileAsync(path, ct);
}
}
@@ -1,4 +1,5 @@
using JdeScoping.ConfigManager.Models;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.ConfigManager.Services;
@@ -15,14 +16,6 @@ public interface IConfigFileService
/// <returns>The loaded configuration model or a new empty model if deserialization fails.</returns>
Task<ConfigModel> LoadAppSettingsAsync(string path, CancellationToken ct = default);
/// <summary>
/// Loads the pipelines configuration from the specified file path.
/// </summary>
/// <param name="path">The file path to load pipelines from.</param>
/// <param name="ct">Cancellation token for the operation.</param>
/// <returns>The loaded pipelines configuration model or a new empty model if deserialization fails.</returns>
Task<PipelinesConfigModel> LoadPipelinesAsync(string path, CancellationToken ct = default);
/// <summary>
/// Saves the application settings configuration to the specified file path.
/// </summary>
@@ -32,10 +25,33 @@ public interface IConfigFileService
Task SaveAppSettingsAsync(string path, ConfigModel config, CancellationToken ct = default);
/// <summary>
/// Saves the pipelines configuration to the specified file path.
/// Loads a single pipeline from a pipeline.*.json file.
/// </summary>
/// <param name="path">The file path to save pipelines to.</param>
/// <param name="config">The pipelines configuration model to save.</param>
/// <param name="path">The file path to the pipeline file.</param>
/// <param name="ct">Cancellation token for the operation.</param>
Task SavePipelinesAsync(string path, PipelinesConfigModel config, CancellationToken ct = default);
/// <returns>The loaded pipeline configuration.</returns>
Task<EtlPipelineConfig> LoadPipelineAsync(string path, CancellationToken ct = default);
/// <summary>
/// Saves a single pipeline to a pipeline.*.json file.
/// </summary>
/// <param name="path">The file path to save the pipeline to.</param>
/// <param name="config">The pipeline configuration to save.</param>
/// <param name="ct">Cancellation token for the operation.</param>
Task SavePipelineAsync(string path, EtlPipelineConfig config, CancellationToken ct = default);
/// <summary>
/// Loads all pipelines from a directory containing pipeline.*.json files.
/// </summary>
/// <param name="directory">The directory containing pipeline files.</param>
/// <param name="ct">Cancellation token for the operation.</param>
/// <returns>A dictionary of pipeline name to configuration.</returns>
Task<Dictionary<string, EtlPipelineConfig>> LoadAllPipelinesAsync(string directory, CancellationToken ct = default);
/// <summary>
/// Deletes a pipeline file.
/// </summary>
/// <param name="path">The file path to delete.</param>
/// <param name="ct">Cancellation token for the operation.</param>
Task DeletePipelineFileAsync(string path, CancellationToken ct = default);
}
@@ -1,4 +1,5 @@
using JdeScoping.ConfigManager.Models;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.ConfigManager.Services;
@@ -48,9 +49,17 @@ public interface IValidationService
ValidationResult ValidateAppSettings(ConfigModel config);
/// <summary>
/// Validates the pipelines configuration.
/// Validates all pipelines in the dictionary.
/// </summary>
/// <param name="config">The pipelines configuration model to validate.</param>
/// <param name="pipelines">Dictionary of pipeline name to configuration.</param>
/// <returns>A validation result containing any errors or warnings found.</returns>
ValidationResult ValidatePipelines(PipelinesConfigModel config);
ValidationResult ValidatePipelines(Dictionary<string, EtlPipelineConfig> pipelines);
/// <summary>
/// Validates a single pipeline configuration.
/// </summary>
/// <param name="pipeline">The pipeline configuration to validate.</param>
/// <param name="name">The pipeline name (used in error messages).</param>
/// <returns>A validation result containing any errors or warnings found.</returns>
ValidationResult ValidatePipeline(EtlPipelineConfig pipeline, string name);
}
@@ -1,4 +1,5 @@
using JdeScoping.ConfigManager.Models;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.ConfigManager.Services;
@@ -56,23 +57,49 @@ public class ValidationService : IValidationService
}
/// <summary>
/// Validates the pipelines configuration.
/// Validates all pipelines in the dictionary.
/// </summary>
/// <param name="config">The pipelines configuration model to validate.</param>
/// <param name="pipelines">Dictionary of pipeline name to configuration.</param>
/// <returns>A validation result containing any errors or warnings found.</returns>
public ValidationResult ValidatePipelines(PipelinesConfigModel config)
public ValidationResult ValidatePipelines(Dictionary<string, EtlPipelineConfig> pipelines)
{
var result = new ValidationResult();
foreach (var (name, pipeline) in config.Pipelines)
foreach (var (name, pipeline) in pipelines)
{
if (string.IsNullOrWhiteSpace(name))
{
result.AddError("Pipeline name cannot be empty");
continue;
}
var pipelineResult = ValidatePipeline(pipeline, name);
foreach (var error in pipelineResult.Errors)
result.AddError(error);
foreach (var warning in pipelineResult.Warnings)
result.AddWarning(warning);
}
// Source validation
return result;
}
/// <summary>
/// Validates a single pipeline configuration.
/// </summary>
/// <param name="pipeline">The pipeline configuration to validate.</param>
/// <param name="name">The pipeline name (used in error messages).</param>
/// <returns>A validation result containing any errors or warnings found.</returns>
public ValidationResult ValidatePipeline(EtlPipelineConfig pipeline, string name)
{
var result = new ValidationResult();
if (string.IsNullOrWhiteSpace(name))
{
result.AddError("Pipeline name cannot be empty");
return result;
}
// Source validation
if (pipeline.Source == null)
{
result.AddError($"Pipeline '{name}': Source is required");
}
else
{
if (string.IsNullOrWhiteSpace(pipeline.Source.Connection))
{
result.AddError($"Pipeline '{name}': Source.Connection is required");
@@ -86,34 +113,55 @@ public class ValidationService : IValidationService
{
result.AddError($"Pipeline '{name}': Source.Query is required");
}
}
// Destination validation
// Destination validation
if (pipeline.Destination == null)
{
result.AddError($"Pipeline '{name}': Destination is required");
}
else
{
if (string.IsNullOrWhiteSpace(pipeline.Destination.Table))
{
result.AddError($"Pipeline '{name}': Destination.Table is required");
}
if (pipeline.Destination.MatchColumns.Length == 0)
if (pipeline.Destination.MatchColumns.Count == 0)
{
result.AddWarning($"Pipeline '{name}': No MatchColumns specified - all rows will be inserted");
}
}
// Schedule validation
ValidateSchedule(result, name, "Mass", pipeline.Schedules.Mass, 60);
ValidateSchedule(result, name, "Daily", pipeline.Schedules.Daily, 60);
ValidateSchedule(result, name, "Hourly", pipeline.Schedules.Hourly, 15);
// Schedule validation - must have at least one schedule unless manual-only
if (!pipeline.IsManualOnly)
{
var hasSchedule = pipeline.MassSyncIntervalMinutes.HasValue ||
pipeline.DailySyncIntervalMinutes.HasValue ||
pipeline.HourlySyncIntervalMinutes.HasValue;
if (!hasSchedule)
{
result.AddWarning($"Pipeline '{name}': No sync schedule configured and not marked as manual-only");
}
// Validate minimum intervals
if (pipeline.MassSyncIntervalMinutes.HasValue && pipeline.MassSyncIntervalMinutes.Value < 60)
{
result.AddError($"Pipeline '{name}': Mass sync interval must be at least 60 minutes");
}
if (pipeline.DailySyncIntervalMinutes.HasValue && pipeline.DailySyncIntervalMinutes.Value < 60)
{
result.AddError($"Pipeline '{name}': Daily sync interval must be at least 60 minutes");
}
if (pipeline.HourlySyncIntervalMinutes.HasValue && pipeline.HourlySyncIntervalMinutes.Value < 15)
{
result.AddError($"Pipeline '{name}': Hourly sync interval must be at least 15 minutes");
}
}
return result;
}
private void ValidateSchedule(ValidationResult result, string pipelineName, string scheduleName, ScheduleModel? schedule, int minInterval)
{
if (schedule == null) return;
if (schedule.Enabled && schedule.IntervalMinutes < minInterval)
{
result.AddError($"Pipeline '{pipelineName}': {scheduleName} schedule interval must be at least {minInterval} minutes");
}
}
}
@@ -1,6 +1,6 @@
using JdeScoping.ConfigManager.Models;
using JdeScoping.ConfigManager.Services;
using JdeScoping.ConfigManager.ViewModels.PipelineSteps;
using JdeScoping.DataSync.Configuration;
using System.Collections.ObjectModel;
using System.Windows.Input;
@@ -11,13 +11,13 @@ namespace JdeScoping.ConfigManager.ViewModels.Forms;
/// </summary>
public class PipelineEditorViewModel : ViewModelBase
{
private readonly PipelineModel _model;
private readonly EtlPipelineConfig _model;
private readonly Action _onChanged;
private readonly IDialogService _dialogService;
private PipelineStepViewModelBase? _selectedStep;
private object? _selectedStepEditor;
public PipelineEditorViewModel(string name, PipelineModel model, IReadOnlyList<string> availableConnections, IDialogService dialogService, Action onChanged)
public PipelineEditorViewModel(string name, EtlPipelineConfig model, IReadOnlyList<string> availableConnections, IDialogService dialogService, Action onChanged)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
_model = model ?? throw new ArgumentNullException(nameof(model));
@@ -30,15 +30,6 @@ public class PipelineEditorViewModel : ViewModelBase
Transformers = [];
PostScripts = [];
// Initialize schedule view models
_model.Schedules.Mass ??= new ScheduleModel();
_model.Schedules.Daily ??= new ScheduleModel();
_model.Schedules.Hourly ??= new ScheduleModel();
MassSchedule = new ScheduleFormViewModel(_model.Schedules.Mass, _onChanged);
DailySchedule = new ScheduleFormViewModel(_model.Schedules.Daily, _onChanged);
HourlySchedule = new ScheduleFormViewModel(_model.Schedules.Hourly, _onChanged);
// Build the pipeline steps from the model
BuildPipelineSteps();
@@ -167,19 +158,164 @@ public class PipelineEditorViewModel : ViewModelBase
}
/// <summary>
/// Gets the mass schedule view model.
/// Gets or sets whether the pipeline is enabled.
/// </summary>
public ScheduleFormViewModel MassSchedule { get; }
public bool IsEnabled
{
get => _model.IsEnabled;
set
{
if (_model.IsEnabled != value)
{
_model.IsEnabled = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets the daily schedule view model.
/// Gets or sets whether the pipeline is manual-only.
/// </summary>
public ScheduleFormViewModel DailySchedule { get; }
public bool IsManualOnly
{
get => _model.IsManualOnly;
set
{
if (_model.IsManualOnly != value)
{
_model.IsManualOnly = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets the hourly schedule view model.
/// Gets or sets whether mass sync is enabled.
/// </summary>
public ScheduleFormViewModel HourlySchedule { get; }
public bool MassSyncEnabled
{
get => _model.MassSyncIntervalMinutes.HasValue;
set
{
if (value && !_model.MassSyncIntervalMinutes.HasValue)
{
_model.MassSyncIntervalMinutes = 10080; // 1 week
OnPropertyChanged();
OnPropertyChanged(nameof(MassSyncIntervalMinutes));
_onChanged();
}
else if (!value && _model.MassSyncIntervalMinutes.HasValue)
{
_model.MassSyncIntervalMinutes = null;
OnPropertyChanged();
OnPropertyChanged(nameof(MassSyncIntervalMinutes));
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the mass sync interval in minutes.
/// </summary>
public int MassSyncIntervalMinutes
{
get => _model.MassSyncIntervalMinutes ?? 10080;
set
{
if (_model.MassSyncIntervalMinutes != value)
{
_model.MassSyncIntervalMinutes = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets whether daily sync is enabled.
/// </summary>
public bool DailySyncEnabled
{
get => _model.DailySyncIntervalMinutes.HasValue;
set
{
if (value && !_model.DailySyncIntervalMinutes.HasValue)
{
_model.DailySyncIntervalMinutes = 1440; // 1 day
OnPropertyChanged();
OnPropertyChanged(nameof(DailySyncIntervalMinutes));
_onChanged();
}
else if (!value && _model.DailySyncIntervalMinutes.HasValue)
{
_model.DailySyncIntervalMinutes = null;
OnPropertyChanged();
OnPropertyChanged(nameof(DailySyncIntervalMinutes));
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the daily sync interval in minutes.
/// </summary>
public int DailySyncIntervalMinutes
{
get => _model.DailySyncIntervalMinutes ?? 1440;
set
{
if (_model.DailySyncIntervalMinutes != value)
{
_model.DailySyncIntervalMinutes = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets whether hourly sync is enabled.
/// </summary>
public bool HourlySyncEnabled
{
get => _model.HourlySyncIntervalMinutes.HasValue;
set
{
if (value && !_model.HourlySyncIntervalMinutes.HasValue)
{
_model.HourlySyncIntervalMinutes = 60; // 1 hour
OnPropertyChanged();
OnPropertyChanged(nameof(HourlySyncIntervalMinutes));
_onChanged();
}
else if (!value && _model.HourlySyncIntervalMinutes.HasValue)
{
_model.HourlySyncIntervalMinutes = null;
OnPropertyChanged();
OnPropertyChanged(nameof(HourlySyncIntervalMinutes));
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the hourly sync interval in minutes.
/// </summary>
public int HourlySyncIntervalMinutes
{
get => _model.HourlySyncIntervalMinutes ?? 60;
set
{
if (_model.HourlySyncIntervalMinutes != value)
{
_model.HourlySyncIntervalMinutes = value;
OnPropertyChanged();
_onChanged();
}
}
}
// Commands
public ICommand AddPreScriptCommand { get; }
@@ -211,16 +347,13 @@ public class PipelineEditorViewModel : ViewModelBase
{
// Pre-scripts
PreScripts.Clear();
if (_model.PreScripts != null)
foreach (var script in _model.PreScripts)
{
foreach (var script in _model.PreScripts)
PreScripts.Add(new PreScriptStepViewModel(script, () =>
{
PreScripts.Add(new PreScriptStepViewModel(script, () =>
{
SyncPreScriptsToModel();
_onChanged();
}));
}
SyncPreScriptsToModel();
_onChanged();
}));
}
// Source
@@ -231,18 +364,15 @@ public class PipelineEditorViewModel : ViewModelBase
// Transformers
Transformers.Clear();
if (_model.Transformers != null)
foreach (var transform in _model.Transforms)
{
foreach (var transformer in _model.Transformers)
var vm = TransformerFactory.Create(transform, () =>
{
var vm = TransformerFactory.Create(transformer, () =>
{
SyncTransformersToModel();
_onChanged();
});
if (vm != null)
Transformers.Add(vm);
}
SyncTransformersToModel();
_onChanged();
});
if (vm != null)
Transformers.Add(vm);
}
// Destination
@@ -253,16 +383,13 @@ public class PipelineEditorViewModel : ViewModelBase
// Post-scripts
PostScripts.Clear();
if (_model.PostScripts != null)
foreach (var script in _model.PostScripts)
{
foreach (var script in _model.PostScripts)
PostScripts.Add(new PostScriptStepViewModel(script, () =>
{
PostScripts.Add(new PostScriptStepViewModel(script, () =>
{
SyncPostScriptsToModel();
_onChanged();
}));
}
SyncPostScriptsToModel();
_onChanged();
}));
}
OnPropertyChanged(nameof(AllSteps));
@@ -277,7 +404,7 @@ public class PipelineEditorViewModel : ViewModelBase
private void AddPreScript()
{
var step = new PreScriptStepViewModel(string.Empty, () =>
var step = new PreScriptStepViewModel(() =>
{
SyncPreScriptsToModel();
_onChanged();
@@ -330,7 +457,7 @@ public class PipelineEditorViewModel : ViewModelBase
private void AddPostScript()
{
var step = new PostScriptStepViewModel(string.Empty, () =>
var step = new PostScriptStepViewModel(() =>
{
SyncPostScriptsToModel();
_onChanged();
@@ -477,23 +604,29 @@ public class PipelineEditorViewModel : ViewModelBase
private void SyncPreScriptsToModel()
{
_model.PreScripts = PreScripts.Count > 0
? PreScripts.Select(s => s.Script).ToArray()
: null;
_model.PreScripts.Clear();
foreach (var script in PreScripts)
{
_model.PreScripts.Add(script.ToModel());
}
}
private void SyncTransformersToModel()
{
_model.Transformers = Transformers.Count > 0
? Transformers.Select(t => t.ToModel()).ToList()
: null;
_model.Transforms.Clear();
foreach (var transformer in Transformers)
{
_model.Transforms.Add(transformer.ToModel());
}
}
private void SyncPostScriptsToModel()
{
_model.PostScripts = PostScripts.Count > 0
? PostScripts.Select(s => s.Script).ToArray()
: null;
_model.PostScripts.Clear();
foreach (var script in PostScripts)
{
_model.PostScripts.Add(script.ToModel());
}
}
/// <summary>
@@ -1,172 +0,0 @@
using JdeScoping.ConfigManager.Models;
namespace JdeScoping.ConfigManager.ViewModels.Forms;
/// <summary>
/// ViewModel for editing a pipeline configuration.
/// </summary>
public class PipelineFormViewModel : ViewModelBase
{
private readonly PipelineModel _model;
private readonly Action _onChanged;
public PipelineFormViewModel(string name, PipelineModel model, Action onChanged)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
_model = model ?? throw new ArgumentNullException(nameof(model));
_onChanged = onChanged ?? throw new ArgumentNullException(nameof(onChanged));
// Initialize schedule view models
_model.Schedules.Mass ??= new ScheduleModel();
_model.Schedules.Daily ??= new ScheduleModel();
_model.Schedules.Hourly ??= new ScheduleModel();
MassSchedule = new ScheduleFormViewModel(_model.Schedules.Mass, _onChanged);
DailySchedule = new ScheduleFormViewModel(_model.Schedules.Daily, _onChanged);
HourlySchedule = new ScheduleFormViewModel(_model.Schedules.Hourly, _onChanged);
}
/// <summary>
/// Gets the pipeline name.
/// </summary>
public string Name { get; }
/// <summary>
/// Gets or sets the source connection name.
/// </summary>
public string Connection
{
get => _model.Source.Connection;
set
{
if (_model.Source.Connection != value)
{
_model.Source.Connection = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the source query.
/// </summary>
public string Query
{
get => _model.Source.Query;
set
{
if (_model.Source.Query != value)
{
_model.Source.Query = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the optional mass query.
/// </summary>
public string? MassQuery
{
get => _model.Source.MassQuery;
set
{
if (_model.Source.MassQuery != value)
{
_model.Source.MassQuery = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the destination table name.
/// </summary>
public string DestinationTable
{
get => _model.Destination.Table;
set
{
if (_model.Destination.Table != value)
{
_model.Destination.Table = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the match columns as newline-separated text.
/// </summary>
public string MatchColumnsText
{
get => string.Join("\n", _model.Destination.MatchColumns);
set
{
var columns = value.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (!_model.Destination.MatchColumns.SequenceEqual(columns))
{
_model.Destination.MatchColumns = columns;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the exclude from update columns as newline-separated text.
/// </summary>
public string ExcludeFromUpdateText
{
get => string.Join("\n", _model.Destination.ExcludeFromUpdate);
set
{
var columns = value.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (!_model.Destination.ExcludeFromUpdate.SequenceEqual(columns))
{
_model.Destination.ExcludeFromUpdate = columns;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the post scripts as newline-separated text.
/// </summary>
public string PostScriptsText
{
get => _model.PostScripts != null ? string.Join("\n", _model.PostScripts) : string.Empty;
set
{
var scripts = string.IsNullOrWhiteSpace(value)
? null
: value.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (_model.PostScripts?.SequenceEqual(scripts ?? []) != true)
{
_model.PostScripts = scripts;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets the mass schedule view model.
/// </summary>
public ScheduleFormViewModel MassSchedule { get; }
/// <summary>
/// Gets the daily schedule view model.
/// </summary>
public ScheduleFormViewModel DailySchedule { get; }
/// <summary>
/// Gets the hourly schedule view model.
/// </summary>
public ScheduleFormViewModel HourlySchedule { get; }
}
@@ -1,86 +0,0 @@
using JdeScoping.ConfigManager.Models;
namespace JdeScoping.ConfigManager.ViewModels.Forms;
/// <summary>
/// ViewModel for editing a schedule configuration.
/// </summary>
public class ScheduleFormViewModel : ViewModelBase
{
private readonly ScheduleModel _model;
private readonly Action _onChanged;
public ScheduleFormViewModel(ScheduleModel model, Action onChanged)
{
_model = model ?? throw new ArgumentNullException(nameof(model));
_onChanged = onChanged ?? throw new ArgumentNullException(nameof(onChanged));
}
/// <summary>
/// Gets or sets whether this schedule is enabled.
/// </summary>
public bool Enabled
{
get => _model.Enabled;
set
{
if (_model.Enabled != value)
{
_model.Enabled = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets the interval in minutes.
/// </summary>
public int IntervalMinutes
{
get => _model.IntervalMinutes;
set
{
if (_model.IntervalMinutes != value)
{
_model.IntervalMinutes = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets whether to purge before sync.
/// </summary>
public bool PrePurge
{
get => _model.PrePurge;
set
{
if (_model.PrePurge != value)
{
_model.PrePurge = value;
OnPropertyChanged();
_onChanged();
}
}
}
/// <summary>
/// Gets or sets whether to reindex after sync.
/// </summary>
public bool ReIndex
{
get => _model.ReIndex;
set
{
if (_model.ReIndex != value)
{
_model.ReIndex = value;
OnPropertyChanged();
_onChanged();
}
}
}
}
@@ -7,6 +7,7 @@ using JdeScoping.ConfigManager.Services;
using JdeScoping.ConfigManager.Services.SecureStore;
using JdeScoping.ConfigManager.ViewModels.Dialogs;
using JdeScoping.ConfigManager.ViewModels.Forms;
using JdeScoping.DataSync.Configuration;
using Microsoft.Extensions.Logging;
namespace JdeScoping.ConfigManager.ViewModels;
@@ -36,7 +37,7 @@ public class MainWindowViewModel : ViewModelBase
private object? _selectedFormViewModel;
private ConfigModel? _appSettings;
private PipelinesConfigModel? _pipelines;
private Dictionary<string, EtlPipelineConfig>? _pipelines;
/// <summary>
/// Gets or sets the currently loaded configuration folder path.
@@ -423,13 +424,17 @@ public class MainWindowViewModel : ViewModelBase
var appSettingsPath = Path.Combine(folderPath, "appsettings.json");
_appSettings = await _configFileService.LoadAppSettingsAsync(appSettingsPath);
// Use config-driven pipeline path
var pipelinesConfigPath = _appSettings?.Pipelines?.ConfigPath ?? "Pipelines/pipelines.json";
var pipelinesPath = Path.Combine(folderPath, pipelinesConfigPath);
// Load pipelines from directory containing pipeline.*.json files
var pipelinesDirectory = Path.Combine(folderPath,
_appSettings?.Pipelines?.ConfigDirectory ?? "Pipelines");
if (File.Exists(pipelinesPath))
if (Directory.Exists(pipelinesDirectory))
{
_pipelines = await _configFileService.LoadPipelinesAsync(pipelinesPath);
_pipelines = await _configFileService.LoadAllPipelinesAsync(pipelinesDirectory);
}
else
{
_pipelines = new Dictionary<string, EtlPipelineConfig>(StringComparer.OrdinalIgnoreCase);
}
// Initialize SecureStore (auto-create if needed, open, sync required keys)
@@ -468,7 +473,7 @@ public class MainWindowViewModel : ViewModelBase
var pipelinesFolder = new TreeNodeViewModel("Pipelines", "⚡", TreeNodeType.Folder) { IsExpanded = true };
if (_pipelines != null)
{
foreach (var (name, _) in _pipelines.Pipelines)
foreach (var name in _pipelines.Keys.OrderBy(k => k))
{
pipelinesFolder.Children.Add(new TreeNodeViewModel(name, "📦", TreeNodeType.Pipeline) { SectionKey = name });
}
@@ -574,7 +579,7 @@ public class MainWindowViewModel : ViewModelBase
_dialogService,
_connectionTestService),
_ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null && _dialogService != null
=> _pipelines.Pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline)
=> _pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline)
? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), _dialogService, MarkAsChanged)
: null,
_ => null
@@ -659,8 +664,8 @@ public class MainWindowViewModel : ViewModelBase
/// Loads configuration for testing purposes.
/// </summary>
/// <param name="appSettings">The application settings configuration model.</param>
/// <param name="pipelines">The pipelines configuration model.</param>
public void LoadConfigForTesting(ConfigModel? appSettings, PipelinesConfigModel? pipelines)
/// <param name="pipelines">The pipelines dictionary.</param>
public void LoadConfigForTesting(ConfigModel? appSettings, Dictionary<string, EtlPipelineConfig>? pipelines)
{
_appSettings = appSettings;
_pipelines = pipelines;
@@ -687,16 +692,19 @@ public class MainWindowViewModel : ViewModelBase
// Save appsettings
await _configFileService.SaveAppSettingsAsync(appSettingsPath, _appSettings);
// Save pipelines if loaded
// Save each pipeline to its own file
if (_pipelines != null)
{
var pipelinesConfigPath = _appSettings?.Pipelines?.ConfigPath ?? "Pipelines/pipelines.json";
var pipelinesPath = Path.Combine(ConfigFolderPath, pipelinesConfigPath);
if (File.Exists(pipelinesPath))
var pipelinesDirectory = Path.Combine(ConfigFolderPath,
_appSettings?.Pipelines?.ConfigDirectory ?? "Pipelines");
Directory.CreateDirectory(pipelinesDirectory);
foreach (var (name, pipeline) in _pipelines)
{
await _backupService.CreateBackupAsync(pipelinesPath);
var filePath = Path.Combine(pipelinesDirectory, $"pipeline.{name}.json");
await _configFileService.SavePipelineAsync(filePath, pipeline);
}
await _configFileService.SavePipelinesAsync(pipelinesPath, _pipelines);
}
HasUnsavedChanges = false;
@@ -834,22 +842,23 @@ public class MainWindowViewModel : ViewModelBase
return;
// Check for duplicate
if (_pipelines.Pipelines.ContainsKey(name))
if (_pipelines.ContainsKey(name))
{
await _dialogService.ShowMessageAsync("Error",
$"Pipeline '{name}' already exists.");
return;
}
// Create default pipeline model
var pipeline = new PipelineModel
// Create default pipeline using EtlPipelineConfig
var pipeline = new EtlPipelineConfig
{
Source = new PipelineSource { Connection = "lotfinder", Query = "" },
Destination = new PipelineDestination { Table = name },
Schedules = new PipelineSchedules()
Name = name,
IsEnabled = true,
Source = new SourceElement { Connection = "lotfinder", Query = "" },
Destination = new DestinationElement { Table = name }
};
_pipelines.Pipelines[name] = pipeline;
_pipelines[name] = pipeline;
// Add tree node
var pipelinesFolder = TreeNodes.FirstOrDefault(n =>
@@ -875,7 +884,8 @@ public class MainWindowViewModel : ViewModelBase
{
if (_selectedNode?.NodeType != TreeNodeType.Pipeline ||
_pipelines == null ||
_dialogService == null)
_dialogService == null ||
_appSettings == null)
return;
var name = _selectedNode.SectionKey!;
@@ -888,7 +898,16 @@ public class MainWindowViewModel : ViewModelBase
return;
// Remove from model
_pipelines.Pipelines.Remove(name);
_pipelines.Remove(name);
// Delete the pipeline file
var pipelinesDirectory = Path.Combine(ConfigFolderPath,
_appSettings.Pipelines?.ConfigDirectory ?? "Pipelines");
var filePath = Path.Combine(pipelinesDirectory, $"pipeline.{name}.json");
if (File.Exists(filePath))
{
await _configFileService.DeletePipelineFileAsync(filePath);
}
// Remove tree node
var pipelinesFolder = TreeNodes.FirstOrDefault(n =>
@@ -1,24 +1,15 @@
using JdeScoping.ConfigManager.Models;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps;
/// <summary>
/// Destination type for the pipeline.
/// </summary>
public enum DestinationType
{
BulkImport,
BulkMerge
}
/// <summary>
/// View model for the destination step in a pipeline.
/// </summary>
public class DestinationStepViewModel : PipelineStepViewModelBase
{
private readonly PipelineDestination _model;
private readonly DestinationElement _model;
public DestinationStepViewModel(PipelineDestination model, Action onChanged) : base(onChanged)
public DestinationStepViewModel(DestinationElement model, Action onChanged) : base(onChanged)
{
_model = model ?? throw new ArgumentNullException(nameof(model));
}
@@ -28,40 +19,6 @@ public class DestinationStepViewModel : PipelineStepViewModelBase
public override string Icon => "󰆼"; // mdi-database
public override string Summary => !string.IsNullOrEmpty(Table) ? $"→ {Table}" : "(no table)";
/// <summary>
/// Gets or sets the destination type (BulkImport or BulkMerge).
/// </summary>
public DestinationType Type
{
get => _model.Type?.Equals("BulkImport", StringComparison.OrdinalIgnoreCase) == true
? DestinationType.BulkImport
: DestinationType.BulkMerge;
set
{
var typeStr = value == DestinationType.BulkImport ? "BulkImport" : "BulkMerge";
if (_model.Type != typeStr)
{
_model.Type = typeStr;
OnPropertyChanged();
OnPropertyChanged(nameof(IsBulkMerge));
OnPropertyChanged(nameof(TypeDescription));
NotifyChanged();
}
}
}
/// <summary>
/// Gets whether the destination type is BulkMerge (shows match columns).
/// </summary>
public bool IsBulkMerge => Type == DestinationType.BulkMerge;
/// <summary>
/// Gets a description of the current type.
/// </summary>
public string TypeDescription => Type == DestinationType.BulkImport
? "Truncate table and bulk load all data"
: "Merge data using match columns (upsert)";
/// <summary>
/// Gets or sets the destination table name.
/// </summary>
@@ -82,14 +39,13 @@ public class DestinationStepViewModel : PipelineStepViewModelBase
/// <summary>
/// Gets or sets the match columns as newline-separated text.
/// Only used for BulkMerge type.
/// </summary>
public string MatchColumnsText
{
get => string.Join("\n", _model.MatchColumns);
set
{
var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList();
if (!_model.MatchColumns.SequenceEqual(columns))
{
_model.MatchColumns = columns;
@@ -101,14 +57,13 @@ public class DestinationStepViewModel : PipelineStepViewModelBase
/// <summary>
/// Gets or sets the columns to exclude from updates as newline-separated text.
/// Only used for BulkMerge type.
/// </summary>
public string ExcludeFromUpdateText
{
get => string.Join("\n", _model.ExcludeFromUpdate);
set
{
var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList();
if (!_model.ExcludeFromUpdate.SequenceEqual(columns))
{
_model.ExcludeFromUpdate = columns;
@@ -1,4 +1,5 @@
using System.Windows.Input;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps;
@@ -70,34 +71,66 @@ public abstract class PipelineStepViewModelBase : ViewModelBase
/// </summary>
public class PreScriptStepViewModel : PipelineStepViewModelBase
{
private string _script;
private readonly ScriptElement _model;
public PreScriptStepViewModel(string script, Action onChanged) : base(onChanged)
public PreScriptStepViewModel(ScriptElement model, Action onChanged) : base(onChanged)
{
_script = script ?? string.Empty;
_model = model ?? throw new ArgumentNullException(nameof(model));
}
/// <summary>
/// Creates a new pre-script with default values.
/// </summary>
public PreScriptStepViewModel(Action onChanged) : base(onChanged)
{
_model = new ScriptElement { Connection = "lotfinder", Script = string.Empty };
}
public override PipelineStepType StepType => PipelineStepType.PreScript;
public override string DisplayName => "Pre-Script";
public override string Icon => "󰯂"; // mdi-script-text
public override string Summary => TruncateScript(_script);
public override string Summary => TruncateScript(_model.Script);
/// <summary>
/// Gets or sets the connection for script execution.
/// </summary>
public string Connection
{
get => _model.Connection;
set
{
if (_model.Connection != value)
{
_model.Connection = value ?? "lotfinder";
OnPropertyChanged();
NotifyChanged();
}
}
}
/// <summary>
/// Gets or sets the SQL script content.
/// </summary>
public string Script
{
get => _script;
get => _model.Script;
set
{
if (SetProperty(ref _script, value ?? string.Empty))
if (_model.Script != value)
{
_model.Script = value ?? string.Empty;
OnPropertyChanged();
OnPropertyChanged(nameof(Summary));
NotifyChanged();
}
}
}
/// <summary>
/// Gets the underlying model.
/// </summary>
public ScriptElement ToModel() => _model;
private static string TruncateScript(string script)
{
if (string.IsNullOrWhiteSpace(script)) return "(empty)";
@@ -111,34 +144,66 @@ public class PreScriptStepViewModel : PipelineStepViewModelBase
/// </summary>
public class PostScriptStepViewModel : PipelineStepViewModelBase
{
private string _script;
private readonly ScriptElement _model;
public PostScriptStepViewModel(string script, Action onChanged) : base(onChanged)
public PostScriptStepViewModel(ScriptElement model, Action onChanged) : base(onChanged)
{
_script = script ?? string.Empty;
_model = model ?? throw new ArgumentNullException(nameof(model));
}
/// <summary>
/// Creates a new post-script with default values.
/// </summary>
public PostScriptStepViewModel(Action onChanged) : base(onChanged)
{
_model = new ScriptElement { Connection = "lotfinder", Script = string.Empty };
}
public override PipelineStepType StepType => PipelineStepType.PostScript;
public override string DisplayName => "Post-Script";
public override string Icon => "󰯂"; // mdi-script-text
public override string Summary => TruncateScript(_script);
public override string Summary => TruncateScript(_model.Script);
/// <summary>
/// Gets or sets the connection for script execution.
/// </summary>
public string Connection
{
get => _model.Connection;
set
{
if (_model.Connection != value)
{
_model.Connection = value ?? "lotfinder";
OnPropertyChanged();
NotifyChanged();
}
}
}
/// <summary>
/// Gets or sets the SQL script content.
/// </summary>
public string Script
{
get => _script;
get => _model.Script;
set
{
if (SetProperty(ref _script, value ?? string.Empty))
if (_model.Script != value)
{
_model.Script = value ?? string.Empty;
OnPropertyChanged();
OnPropertyChanged(nameof(Summary));
NotifyChanged();
}
}
}
/// <summary>
/// Gets the underlying model.
/// </summary>
public ScriptElement ToModel() => _model;
private static string TruncateScript(string script)
{
if (string.IsNullOrWhiteSpace(script)) return "(empty)";
@@ -1,26 +1,17 @@
using JdeScoping.ConfigManager.Models;
using JdeScoping.DataSync.Configuration;
using System.Collections.ObjectModel;
using System.Windows.Input;
namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps;
/// <summary>
/// Source type for the pipeline.
/// </summary>
public enum SourceType
{
Database,
File
}
/// <summary>
/// View model for the source step in a pipeline.
/// </summary>
public class SourceStepViewModel : PipelineStepViewModelBase
{
private readonly PipelineSource _model;
private readonly SourceElement _model;
public SourceStepViewModel(PipelineSource model, IReadOnlyList<string> availableConnections, Action onChanged) : base(onChanged)
public SourceStepViewModel(SourceElement model, IReadOnlyList<string> availableConnections, Action onChanged) : base(onChanged)
{
_model = model ?? throw new ArgumentNullException(nameof(model));
AvailableConnections = availableConnections ?? [];
@@ -44,54 +35,8 @@ public class SourceStepViewModel : PipelineStepViewModelBase
public override PipelineStepType StepType => PipelineStepType.Source;
public override string DisplayName => "Source";
public override string Icon => IsFileSource ? "󰈔" : "󰆼"; // mdi-file vs mdi-database
public override string Summary => IsFileSource
? $"File: {System.IO.Path.GetFileName(FileName) ?? "(none)"}"
: $"{Connection}: {TruncateQuery(Query)}";
/// <summary>
/// Gets whether this is a file-based source.
/// </summary>
public bool IsFileSource => !string.IsNullOrEmpty(_model.FileName);
/// <summary>
/// Gets whether this is a database source.
/// </summary>
public bool IsDatabaseSource => !IsFileSource;
/// <summary>
/// Gets or sets the source type (Database or File).
/// </summary>
public SourceType SourceType
{
get => IsFileSource ? SourceType.File : SourceType.Database;
set
{
if (value == SourceType.File && !IsFileSource)
{
// Switching to file source
_model.FileName = string.Empty;
_model.Connection = string.Empty;
_model.Query = string.Empty;
_model.MassQuery = null;
OnPropertyChanged();
OnPropertyChanged(nameof(IsFileSource));
OnPropertyChanged(nameof(IsDatabaseSource));
OnPropertyChanged(nameof(Summary));
NotifyChanged();
}
else if (value == SourceType.Database && IsFileSource)
{
// Switching to database source
_model.FileName = null;
OnPropertyChanged();
OnPropertyChanged(nameof(IsFileSource));
OnPropertyChanged(nameof(IsDatabaseSource));
OnPropertyChanged(nameof(Summary));
NotifyChanged();
}
}
}
public override string Icon => "󰆼"; // mdi-database
public override string Summary => $"{Connection}: {TruncateQuery(Query)}";
/// <summary>
/// Gets or sets the source database connection name.
@@ -146,24 +91,6 @@ public class SourceStepViewModel : PipelineStepViewModelBase
}
}
/// <summary>
/// Gets or sets the file name for file-based sources.
/// </summary>
public string? FileName
{
get => _model.FileName;
set
{
if (_model.FileName != value)
{
_model.FileName = value;
OnPropertyChanged();
OnPropertyChanged(nameof(Summary));
NotifyChanged();
}
}
}
/// <summary>
/// Gets the collection of query parameters.
/// </summary>
@@ -180,7 +107,7 @@ public class SourceStepViewModel : PipelineStepViewModelBase
public void AddParameter()
{
var key = $"param{Parameters.Count + 1}";
var param = new ParameterDefinition { Name = key };
var param = new ParameterElement { Name = key, Source = "offset" };
var vm = new ParameterViewModel(key, param, () =>
{
SyncParametersToModel();
@@ -228,15 +155,17 @@ public class ParameterViewModel : ViewModelBase
private string _key;
private string _name;
private string? _format;
private string? _source;
private string _source;
private string? _value;
private readonly Action _onChanged;
public ParameterViewModel(string key, ParameterDefinition model, Action onChanged)
public ParameterViewModel(string key, ParameterElement model, Action onChanged)
{
_key = key;
_name = model.Name;
_format = model.Format;
_source = model.Source;
_value = model.Value;
_onChanged = onChanged;
}
@@ -282,12 +211,25 @@ public class ParameterViewModel : ViewModelBase
/// <summary>
/// Gets or sets the parameter source (e.g., offset, static).
/// </summary>
public string? Source
public string Source
{
get => _source;
set
{
if (SetProperty(ref _source, value))
if (SetProperty(ref _source, value ?? "offset"))
_onChanged();
}
}
/// <summary>
/// Gets or sets the static value (if source is not offset).
/// </summary>
public string? Value
{
get => _value;
set
{
if (SetProperty(ref _value, value))
_onChanged();
}
}
@@ -295,10 +237,11 @@ public class ParameterViewModel : ViewModelBase
/// <summary>
/// Converts this view model back to a model.
/// </summary>
public ParameterDefinition ToModel() => new()
public ParameterElement ToModel() => new()
{
Name = _name,
Format = _format,
Source = _source
Source = _source,
Value = _value
};
}
@@ -1,5 +1,6 @@
using JdeScoping.ConfigManager.Models;
using JdeScoping.DataSync.Configuration;
using System.Collections.ObjectModel;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Windows.Input;
@@ -10,6 +11,11 @@ namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps;
/// </summary>
public abstract class TransformerStepViewModelBase : PipelineStepViewModelBase
{
protected static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
protected TransformerStepViewModelBase(Action onChanged) : base(onChanged)
{
}
@@ -23,9 +29,19 @@ public abstract class TransformerStepViewModelBase : PipelineStepViewModelBase
public abstract string TransformerType { get; }
/// <summary>
/// Converts this view model back to a model.
/// Converts this view model back to a TransformElement.
/// </summary>
public abstract TransformerModel ToModel();
public abstract TransformElement ToModel();
/// <summary>
/// Helper to create a JsonElement from an object.
/// </summary>
protected static JsonElement CreateConfigElement(object config)
{
var json = JsonSerializer.Serialize(config, JsonOptions);
using var doc = JsonDocument.Parse(json);
return doc.RootElement.Clone();
}
}
/// <summary>
@@ -35,9 +51,18 @@ public class ColumnDropTransformerViewModel : TransformerStepViewModelBase
{
private string _columnsText;
public ColumnDropTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged)
public ColumnDropTransformerViewModel(TransformElement element, Action onChanged) : base(onChanged)
{
_columnsText = model.Columns != null ? string.Join("\n", model.Columns) : string.Empty;
_columnsText = string.Empty;
if (element.Config.HasValue)
{
if (element.Config.Value.TryGetProperty("columns", out var columnsProp) &&
columnsProp.ValueKind == JsonValueKind.Array)
{
var columns = columnsProp.EnumerateArray().Select(c => c.GetString() ?? "").Where(c => !string.IsNullOrEmpty(c));
_columnsText = string.Join("\n", columns);
}
}
}
public ColumnDropTransformerViewModel(Action onChanged) : base(onChanged)
@@ -78,10 +103,10 @@ public class ColumnDropTransformerViewModel : TransformerStepViewModelBase
return _columnsText.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Length;
}
public override TransformerModel ToModel() => new()
public override TransformElement ToModel() => new()
{
Type = TransformerType,
Columns = GetColumns()
TransformType = TransformerType,
Config = CreateConfigElement(new { columns = GetColumns() })
};
}
@@ -90,14 +115,25 @@ public class ColumnDropTransformerViewModel : TransformerStepViewModelBase
/// </summary>
public class ColumnRenameTransformerViewModel : TransformerStepViewModelBase
{
public ColumnRenameTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged)
public ColumnRenameTransformerViewModel(TransformElement element, Action onChanged) : base(onChanged)
{
Mappings = new ObservableCollection<ColumnMappingViewModel>(
model.Mappings?.Select(kvp => new ColumnMappingViewModel(kvp.Key, kvp.Value, () =>
Mappings = [];
if (element.Config.HasValue)
{
if (element.Config.Value.TryGetProperty("mappings", out var mappingsProp) &&
mappingsProp.ValueKind == JsonValueKind.Object)
{
OnPropertyChanged(nameof(Summary));
NotifyChanged();
})) ?? []);
foreach (var prop in mappingsProp.EnumerateObject())
{
var newName = prop.Value.GetString() ?? "";
Mappings.Add(new ColumnMappingViewModel(prop.Name, newName, () =>
{
OnPropertyChanged(nameof(Summary));
NotifyChanged();
}));
}
}
}
AddMappingCommand = new RelayCommand(AddMapping);
}
@@ -147,10 +183,10 @@ public class ColumnRenameTransformerViewModel : TransformerStepViewModelBase
}
}
public override TransformerModel ToModel() => new()
public override TransformElement ToModel() => new()
{
Type = TransformerType,
Mappings = Mappings.ToDictionary(m => m.OldName, m => m.NewName)
TransformType = TransformerType,
Config = CreateConfigElement(new { mappings = Mappings.ToDictionary(m => m.OldName, m => m.NewName) })
};
}
@@ -206,11 +242,21 @@ public class JdeDateTransformerViewModel : TransformerStepViewModelBase
private string? _timeColumn;
private string? _outputColumn;
public JdeDateTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged)
public JdeDateTransformerViewModel(TransformElement element, Action onChanged) : base(onChanged)
{
_dateColumn = model.DateColumn;
_timeColumn = model.TimeColumn;
_outputColumn = model.OutputColumn;
_dateColumn = null;
_timeColumn = null;
_outputColumn = null;
if (element.Config.HasValue)
{
if (element.Config.Value.TryGetProperty("dateColumn", out var dateProp))
_dateColumn = dateProp.GetString();
if (element.Config.Value.TryGetProperty("timeColumn", out var timeProp))
_timeColumn = timeProp.GetString();
if (element.Config.Value.TryGetProperty("outputColumn", out var outputProp))
_outputColumn = outputProp.GetString();
}
}
public JdeDateTransformerViewModel(Action onChanged) : base(onChanged)
@@ -270,15 +316,26 @@ public class JdeDateTransformerViewModel : TransformerStepViewModelBase
}
}
public override TransformerModel ToModel() => new()
public override TransformElement ToModel() => new()
{
Type = TransformerType,
DateColumn = _dateColumn,
TimeColumn = _timeColumn,
OutputColumn = _outputColumn
TransformType = TransformerType,
Config = CreateConfigElement(new { dateColumn = _dateColumn, timeColumn = _timeColumn, outputColumn = _outputColumn })
};
}
/// <summary>
/// Specifies behavior when a regex pattern does not match the input value.
/// </summary>
public enum NonMatchBehavior
{
/// <summary>Keep the original value unchanged.</summary>
KeepOriginal,
/// <summary>Return null/DBNull.</summary>
ReturnNull,
/// <summary>Return an empty string.</summary>
ReturnEmpty
}
/// <summary>
/// View model for Regex transformer.
/// </summary>
@@ -301,14 +358,41 @@ public class RegexTransformerViewModel : TransformerStepViewModelBase
private bool _hasTestError;
private string _testErrorMessage = string.Empty;
public RegexTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged)
public RegexTransformerViewModel(TransformElement element, Action onChanged) : base(onChanged)
{
_columnName = model.ColumnName ?? string.Empty;
_pattern = model.Pattern ?? string.Empty;
_replacement = model.Replacement;
_isFindReplaceMode = model.Replacement != null;
_ignoreCase = model.IgnoreCase;
_nonMatchBehavior = model.NonMatchBehavior;
_columnName = string.Empty;
_pattern = string.Empty;
_replacement = null;
_isFindReplaceMode = true;
_ignoreCase = false;
_nonMatchBehavior = NonMatchBehavior.KeepOriginal;
if (element.Config.HasValue)
{
if (element.Config.Value.TryGetProperty("columnName", out var colProp))
_columnName = colProp.GetString() ?? string.Empty;
if (element.Config.Value.TryGetProperty("pattern", out var patternProp))
_pattern = patternProp.GetString() ?? string.Empty;
if (element.Config.Value.TryGetProperty("replacement", out var replaceProp))
{
_replacement = replaceProp.ValueKind == JsonValueKind.Null ? null : replaceProp.GetString();
_isFindReplaceMode = _replacement != null;
}
else
{
// No replacement property means Match & Extract mode
_replacement = null;
_isFindReplaceMode = false;
}
if (element.Config.Value.TryGetProperty("ignoreCase", out var ignoreProp))
_ignoreCase = ignoreProp.GetBoolean();
if (element.Config.Value.TryGetProperty("nonMatchBehavior", out var behaviorProp))
{
var behaviorStr = behaviorProp.GetString();
if (Enum.TryParse<NonMatchBehavior>(behaviorStr, true, out var behavior))
_nonMatchBehavior = behavior;
}
}
TestPatternCommand = new RelayCommand(ExecuteTestPattern);
}
@@ -544,14 +628,17 @@ public class RegexTransformerViewModel : TransformerStepViewModelBase
TestErrorMessage = string.Empty;
}
public override TransformerModel ToModel() => new()
public override TransformElement ToModel() => new()
{
Type = TransformerType,
ColumnName = _columnName,
Pattern = _pattern,
Replacement = _isFindReplaceMode ? _replacement : null,
IgnoreCase = _ignoreCase,
NonMatchBehavior = _nonMatchBehavior
TransformType = TransformerType,
Config = CreateConfigElement(new
{
columnName = _columnName,
pattern = _pattern,
replacement = _isFindReplaceMode ? _replacement : null,
ignoreCase = _ignoreCase,
nonMatchBehavior = _nonMatchBehavior.ToString()
})
};
}
@@ -561,16 +648,16 @@ public class RegexTransformerViewModel : TransformerStepViewModelBase
public static class TransformerFactory
{
/// <summary>
/// Creates a transformer view model from a model.
/// Creates a transformer view model from a TransformElement.
/// </summary>
public static TransformerStepViewModelBase? Create(TransformerModel model, Action onChanged)
public static TransformerStepViewModelBase? Create(TransformElement element, Action onChanged)
{
return model.Type?.ToLowerInvariant() switch
return element.TransformType?.ToLowerInvariant() switch
{
"columndrop" => new ColumnDropTransformerViewModel(model, onChanged),
"columnrename" => new ColumnRenameTransformerViewModel(model, onChanged),
"jdedate" => new JdeDateTransformerViewModel(model, onChanged),
"regex" => new RegexTransformerViewModel(model, onChanged),
"columndrop" => new ColumnDropTransformerViewModel(element, onChanged),
"columnrename" => new ColumnRenameTransformerViewModel(element, onChanged),
"jdedate" => new JdeDateTransformerViewModel(element, onChanged),
"regex" => new RegexTransformerViewModel(element, onChanged),
_ => null // Unknown transformer type
};
}
@@ -13,25 +13,6 @@
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Destination Type Toggle -->
<StackPanel Spacing="4">
<TextBlock Text="Load Type" Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<StackPanel Orientation="Horizontal" Spacing="12">
<RadioButton GroupName="DestType"
IsChecked="{Binding IsBulkMerge}"
Foreground="#E6EDF5">
<TextBlock Text="Bulk Merge (Upsert)" FontSize="12"/>
</RadioButton>
<RadioButton GroupName="DestType"
IsChecked="{Binding !IsBulkMerge}"
Foreground="#E6EDF5">
<TextBlock Text="Bulk Import (Truncate+Load)" FontSize="12"/>
</RadioButton>
</StackPanel>
<TextBlock Text="{Binding TypeDescription}"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Destination Table -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
@@ -48,55 +29,39 @@
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- BulkMerge-specific fields -->
<StackPanel Spacing="16" IsVisible="{Binding IsBulkMerge}">
<!-- Match Columns -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Match Columns (one per line)"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding MatchColumnsText}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="11"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="80"
Watermark="OrderNumber&#x0a;OrderType"/>
<TextBlock Text="Columns to match source rows with existing destination rows"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Exclude From Update -->
<StackPanel Spacing="4">
<TextBlock Text="Exclude From Update (one per line)"
<!-- Match Columns -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Match Columns (one per line)"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBox Text="{Binding ExcludeFromUpdateText}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="11"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="60"
Watermark="CreatedDate&#x0a;CreatedBy"/>
<TextBlock Text="Columns to skip when updating existing rows"
Foreground="#5C6A7A" FontSize="11"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding MatchColumnsText}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="11"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="80"
Watermark="OrderNumber&#x0a;OrderType"/>
<TextBlock Text="Columns to match source rows with existing destination rows (for upsert)"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- BulkImport info box -->
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
CornerRadius="4" Padding="12" Margin="0,8,0,0"
IsVisible="{Binding !IsBulkMerge}">
<StackPanel Spacing="4">
<TextBlock Text="Bulk Import Mode" Foreground="#F59E0B" FontSize="11" FontWeight="Medium"/>
<TextBlock Text="All existing data will be deleted before loading new data."
Foreground="#5C6A7A" FontSize="10"/>
<TextBlock Text="Use this for full table refreshes during mass sync operations."
Foreground="#5C6A7A" FontSize="10"/>
</StackPanel>
</Border>
<!-- Exclude From Update -->
<StackPanel Spacing="4">
<TextBlock Text="Exclude From Update (one per line)"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBox Text="{Binding ExcludeFromUpdateText}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="11"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="60"
Watermark="CreatedDate&#x0a;CreatedBy"/>
<TextBlock Text="Columns to skip when updating existing rows"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
</StackPanel>
</UserControl>
@@ -13,145 +13,106 @@
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Source Type Toggle -->
<!-- Connection -->
<StackPanel Spacing="4">
<TextBlock Text="Source Type" Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<StackPanel Orientation="Horizontal" Spacing="12">
<RadioButton GroupName="SourceType"
IsChecked="{Binding IsDatabaseSource}"
Foreground="#E6EDF5">
<TextBlock Text="Database" FontSize="12"/>
</RadioButton>
<RadioButton GroupName="SourceType"
IsChecked="{Binding IsFileSource}"
Foreground="#E6EDF5">
<TextBlock Text="File" FontSize="12"/>
</RadioButton>
</StackPanel>
</StackPanel>
<!-- Database Source Fields -->
<StackPanel Spacing="16" IsVisible="{Binding IsDatabaseSource}">
<!-- Connection -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Connection"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<ComboBox ItemsSource="{Binding AvailableConnections}"
SelectedItem="{Binding Connection}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="36"
HorizontalAlignment="Stretch"
PlaceholderText="Select connection..."/>
<TextBlock Text="Connection string name from Settings > ConnectionStrings"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Query -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Query"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding Query}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="11"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="100"
Watermark="SELECT ... FROM ... WHERE ..."/>
<TextBlock Text="SQL query for incremental updates (use @LastSync parameter)"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Mass Query -->
<StackPanel Spacing="4">
<TextBlock Text="Mass Query (Optional)"
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Connection"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBox Text="{Binding MassQuery}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="11"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="80"
Watermark="SELECT ... FROM ... (no date filter)"/>
<TextBlock Text="Query for full table reload during mass sync"
Foreground="#5C6A7A" FontSize="11"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<!-- Parameters Section -->
<Expander IsExpanded="False">
<Expander.Header>
<StackPanel Orientation="Horizontal" Spacing="8">
<TextBlock Text="Parameters" Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="{Binding Parameters.Count, StringFormat='({0})'}"
Foreground="#5C6A7A" FontSize="12"/>
</StackPanel>
</Expander.Header>
<StackPanel Spacing="8" Margin="0,8,0,0">
<ItemsControl ItemsSource="{Binding Parameters}">
<ItemsControl.ItemTemplate>
<DataTemplate DataType="steps:ParameterViewModel">
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
CornerRadius="4" Padding="8" Margin="0,0,0,4">
<Grid ColumnDefinitions="*,8,*,8,*,8,Auto">
<StackPanel Grid.Column="0" Spacing="2">
<TextBlock Text="Key" Foreground="#5C6A7A" FontSize="10"/>
<TextBox Text="{Binding Key}"
Background="#232A35" Height="28" FontSize="11"/>
</StackPanel>
<StackPanel Grid.Column="2" Spacing="2">
<TextBlock Text="Format" Foreground="#5C6A7A" FontSize="10"/>
<TextBox Text="{Binding Format}"
Background="#232A35" Height="28" FontSize="11"
Watermark="jdeJulian"/>
</StackPanel>
<StackPanel Grid.Column="4" Spacing="2">
<TextBlock Text="Source" Foreground="#5C6A7A" FontSize="10"/>
<TextBox Text="{Binding Source}"
Background="#232A35" Height="28" FontSize="11"
Watermark="offset"/>
</StackPanel>
<Button Grid.Column="6" Content="X"
Background="Transparent" Foreground="#FF6B6B"
BorderThickness="0" FontSize="11" Width="24"
VerticalAlignment="Bottom"/>
</Grid>
</Border>
</DataTemplate>
</ItemsControl.ItemTemplate>
</ItemsControl>
<Button Content="+ Add Parameter"
Background="#232A35" Foreground="#9BA8B8"
BorderBrush="#3D4550" Height="32"
HorizontalAlignment="Left" Padding="12,0"
Command="{Binding AddParameterCommand}"/>
</StackPanel>
</Expander>
<ComboBox ItemsSource="{Binding AvailableConnections}"
SelectedItem="{Binding Connection}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="36"
HorizontalAlignment="Stretch"
PlaceholderText="Select connection..."/>
<TextBlock Text="Connection string name from Settings > ConnectionStrings"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- File Source Fields -->
<StackPanel Spacing="16" IsVisible="{Binding IsFileSource}">
<!-- File Name -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="File Name"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding FileName}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="36"
FontFamily="JetBrains Mono"
Watermark="data.pb.zstd"/>
<TextBlock Text="Protobuf+Zstd compressed file name"
Foreground="#5C6A7A" FontSize="11"/>
<!-- Query -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Query"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding Query}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="11"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="100"
Watermark="SELECT ... FROM ... WHERE ..."/>
<TextBlock Text="SQL query for incremental updates (use @LastSync parameter)"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Mass Query -->
<StackPanel Spacing="4">
<TextBlock Text="Mass Query (Optional)"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBox Text="{Binding MassQuery}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="11"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="80"
Watermark="SELECT ... FROM ... (no date filter)"/>
<TextBlock Text="Query for full table reload during mass sync"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Parameters Section -->
<Expander IsExpanded="False">
<Expander.Header>
<StackPanel Orientation="Horizontal" Spacing="8">
<TextBlock Text="Parameters" Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="{Binding Parameters.Count, StringFormat='({0})'}"
Foreground="#5C6A7A" FontSize="12"/>
</StackPanel>
</Expander.Header>
<StackPanel Spacing="8" Margin="0,8,0,0">
<ItemsControl ItemsSource="{Binding Parameters}">
<ItemsControl.ItemTemplate>
<DataTemplate DataType="steps:ParameterViewModel">
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
CornerRadius="4" Padding="8" Margin="0,0,0,4">
<Grid ColumnDefinitions="*,8,*,8,*,8,Auto">
<StackPanel Grid.Column="0" Spacing="2">
<TextBlock Text="Key" Foreground="#5C6A7A" FontSize="10"/>
<TextBox Text="{Binding Key}"
Background="#232A35" Height="28" FontSize="11"/>
</StackPanel>
<StackPanel Grid.Column="2" Spacing="2">
<TextBlock Text="Format" Foreground="#5C6A7A" FontSize="10"/>
<TextBox Text="{Binding Format}"
Background="#232A35" Height="28" FontSize="11"
Watermark="jdeJulian"/>
</StackPanel>
<StackPanel Grid.Column="4" Spacing="2">
<TextBlock Text="Source" Foreground="#5C6A7A" FontSize="10"/>
<TextBox Text="{Binding Source}"
Background="#232A35" Height="28" FontSize="11"
Watermark="offset"/>
</StackPanel>
<Button Grid.Column="6" Content="X"
Background="Transparent" Foreground="#FF6B6B"
BorderThickness="0" FontSize="11" Width="24"
VerticalAlignment="Bottom"/>
</Grid>
</Border>
</DataTemplate>
</ItemsControl.ItemTemplate>
</ItemsControl>
<Button Content="+ Add Parameter"
Background="#232A35" Foreground="#9BA8B8"
BorderBrush="#3D4550" Height="32"
HorizontalAlignment="Left" Padding="12,0"
Command="{Binding AddParameterCommand}"/>
</StackPanel>
</Expander>
</StackPanel>
</UserControl>
@@ -90,30 +90,30 @@
</Expander.Header>
<StackPanel Spacing="8" Margin="0,8,0,0">
<StackPanel Orientation="Horizontal" Spacing="8">
<CheckBox IsChecked="{Binding MassSchedule.Enabled}"/>
<CheckBox IsChecked="{Binding MassSyncEnabled}"/>
<TextBlock Text="Mass" Foreground="#9BA8B8" FontSize="12"/>
<NumericUpDown Value="{Binding MassSchedule.IntervalMinutes}"
<NumericUpDown Value="{Binding MassSyncIntervalMinutes}"
Minimum="1" Width="80" Height="28"
Background="#232A35" FontSize="11"
IsEnabled="{Binding MassSchedule.Enabled}"/>
IsEnabled="{Binding MassSyncEnabled}"/>
<TextBlock Text="min" Foreground="#5C6A7A" FontSize="11" VerticalAlignment="Center"/>
</StackPanel>
<StackPanel Orientation="Horizontal" Spacing="8">
<CheckBox IsChecked="{Binding DailySchedule.Enabled}"/>
<CheckBox IsChecked="{Binding DailySyncEnabled}"/>
<TextBlock Text="Daily" Foreground="#9BA8B8" FontSize="12"/>
<NumericUpDown Value="{Binding DailySchedule.IntervalMinutes}"
<NumericUpDown Value="{Binding DailySyncIntervalMinutes}"
Minimum="1" Width="80" Height="28"
Background="#232A35" FontSize="11"
IsEnabled="{Binding DailySchedule.Enabled}"/>
IsEnabled="{Binding DailySyncEnabled}"/>
<TextBlock Text="min" Foreground="#5C6A7A" FontSize="11" VerticalAlignment="Center"/>
</StackPanel>
<StackPanel Orientation="Horizontal" Spacing="8">
<CheckBox IsChecked="{Binding HourlySchedule.Enabled}"/>
<CheckBox IsChecked="{Binding HourlySyncEnabled}"/>
<TextBlock Text="Hourly" Foreground="#9BA8B8" FontSize="12"/>
<NumericUpDown Value="{Binding HourlySchedule.IntervalMinutes}"
<NumericUpDown Value="{Binding HourlySyncIntervalMinutes}"
Minimum="1" Width="80" Height="28"
Background="#232A35" FontSize="11"
IsEnabled="{Binding HourlySchedule.Enabled}"/>
IsEnabled="{Binding HourlySyncEnabled}"/>
<TextBlock Text="min" Foreground="#5C6A7A" FontSize="11" VerticalAlignment="Center"/>
</StackPanel>
</StackPanel>
@@ -1,261 +0,0 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:vm="using:JdeScoping.ConfigManager.ViewModels.Forms"
x:Class="JdeScoping.ConfigManager.Views.Forms.PipelineFormView"
x:DataType="vm:PipelineFormViewModel">
<ScrollViewer VerticalScrollBarVisibility="Auto">
<StackPanel Spacing="16" MaxWidth="700">
<!-- Header -->
<StackPanel>
<TextBlock Text="{Binding Name, StringFormat='{}{0} Pipeline'}"
Foreground="#E6EDF5" FontSize="18" FontWeight="SemiBold"/>
<Border Height="1" Background="#2D3540" Margin="0,12,0,0"/>
</StackPanel>
<!-- Source Section (Expander) -->
<Expander IsExpanded="True">
<Expander.Header>
<TextBlock Text="Source" Foreground="#E6EDF5" FontWeight="SemiBold" FontSize="14"/>
</Expander.Header>
<Border Background="#0D0F12" BorderBrush="#2D3540" BorderThickness="1,0,1,1"
CornerRadius="0,0,6,6" Padding="16">
<StackPanel Spacing="16">
<!-- Connection -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Connection"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding Connection}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="36"
FontFamily="JetBrains Mono"
Watermark="JdeOracle"/>
<TextBlock Text="Name of the connection string to use"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Query -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Query"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding Query}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="12"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="120"
Watermark="SELECT ... FROM ... WHERE ..."/>
<TextBlock Text="SQL query for incremental updates (use @LastSync parameter)"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Mass Query -->
<StackPanel Spacing="4">
<TextBlock Text="Mass Query (Optional)"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBox Text="{Binding MassQuery}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="12"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="80"
Watermark="SELECT ... FROM ... (no date filter)"/>
<TextBlock Text="Alternative query for mass sync (full table reload)"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
</StackPanel>
</Border>
</Expander>
<!-- Schedules Section (Expander) -->
<Expander IsExpanded="False">
<Expander.Header>
<TextBlock Text="Schedules" Foreground="#E6EDF5" FontWeight="SemiBold" FontSize="14"/>
</Expander.Header>
<Border Background="#0D0F12" BorderBrush="#2D3540" BorderThickness="1,0,1,1"
CornerRadius="0,0,6,6" Padding="16">
<StackPanel Spacing="20">
<!-- Mass Schedule -->
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
CornerRadius="4" Padding="12">
<StackPanel Spacing="12">
<StackPanel Orientation="Horizontal" Spacing="8">
<CheckBox IsChecked="{Binding MassSchedule.Enabled}"/>
<TextBlock Text="Mass Schedule" Foreground="#E6EDF5" FontWeight="Medium"/>
</StackPanel>
<Grid ColumnDefinitions="*,12,*,12,*" IsEnabled="{Binding MassSchedule.Enabled}">
<StackPanel Grid.Column="0" Spacing="4">
<TextBlock Text="Interval (min)" Foreground="#9BA8B8" FontSize="11"/>
<NumericUpDown Value="{Binding MassSchedule.IntervalMinutes}"
Minimum="1" Maximum="10080"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="32"
FontFamily="JetBrains Mono" FontSize="12"/>
</StackPanel>
<StackPanel Grid.Column="2" Spacing="4">
<CheckBox IsChecked="{Binding MassSchedule.PrePurge}" Foreground="#9BA8B8">
<TextBlock Text="Pre-Purge" Foreground="#9BA8B8" FontSize="11"/>
</CheckBox>
</StackPanel>
<StackPanel Grid.Column="4" Spacing="4">
<CheckBox IsChecked="{Binding MassSchedule.ReIndex}" Foreground="#9BA8B8">
<TextBlock Text="Re-Index" Foreground="#9BA8B8" FontSize="11"/>
</CheckBox>
</StackPanel>
</Grid>
</StackPanel>
</Border>
<!-- Daily Schedule -->
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
CornerRadius="4" Padding="12">
<StackPanel Spacing="12">
<StackPanel Orientation="Horizontal" Spacing="8">
<CheckBox IsChecked="{Binding DailySchedule.Enabled}"/>
<TextBlock Text="Daily Schedule" Foreground="#E6EDF5" FontWeight="Medium"/>
</StackPanel>
<Grid ColumnDefinitions="*,12,*,12,*" IsEnabled="{Binding DailySchedule.Enabled}">
<StackPanel Grid.Column="0" Spacing="4">
<TextBlock Text="Interval (min)" Foreground="#9BA8B8" FontSize="11"/>
<NumericUpDown Value="{Binding DailySchedule.IntervalMinutes}"
Minimum="1" Maximum="1440"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="32"
FontFamily="JetBrains Mono" FontSize="12"/>
</StackPanel>
<StackPanel Grid.Column="2" Spacing="4">
<CheckBox IsChecked="{Binding DailySchedule.PrePurge}" Foreground="#9BA8B8">
<TextBlock Text="Pre-Purge" Foreground="#9BA8B8" FontSize="11"/>
</CheckBox>
</StackPanel>
<StackPanel Grid.Column="4" Spacing="4">
<CheckBox IsChecked="{Binding DailySchedule.ReIndex}" Foreground="#9BA8B8">
<TextBlock Text="Re-Index" Foreground="#9BA8B8" FontSize="11"/>
</CheckBox>
</StackPanel>
</Grid>
</StackPanel>
</Border>
<!-- Hourly Schedule -->
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
CornerRadius="4" Padding="12">
<StackPanel Spacing="12">
<StackPanel Orientation="Horizontal" Spacing="8">
<CheckBox IsChecked="{Binding HourlySchedule.Enabled}"/>
<TextBlock Text="Hourly Schedule" Foreground="#E6EDF5" FontWeight="Medium"/>
</StackPanel>
<Grid ColumnDefinitions="*,12,*,12,*" IsEnabled="{Binding HourlySchedule.Enabled}">
<StackPanel Grid.Column="0" Spacing="4">
<TextBlock Text="Interval (min)" Foreground="#9BA8B8" FontSize="11"/>
<NumericUpDown Value="{Binding HourlySchedule.IntervalMinutes}"
Minimum="1" Maximum="60"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="32"
FontFamily="JetBrains Mono" FontSize="12"/>
</StackPanel>
<StackPanel Grid.Column="2" Spacing="4">
<CheckBox IsChecked="{Binding HourlySchedule.PrePurge}" Foreground="#9BA8B8">
<TextBlock Text="Pre-Purge" Foreground="#9BA8B8" FontSize="11"/>
</CheckBox>
</StackPanel>
<StackPanel Grid.Column="4" Spacing="4">
<CheckBox IsChecked="{Binding HourlySchedule.ReIndex}" Foreground="#9BA8B8">
<TextBlock Text="Re-Index" Foreground="#9BA8B8" FontSize="11"/>
</CheckBox>
</StackPanel>
</Grid>
</StackPanel>
</Border>
</StackPanel>
</Border>
</Expander>
<!-- Destination Section (Expander) -->
<Expander IsExpanded="False">
<Expander.Header>
<TextBlock Text="Destination" Foreground="#E6EDF5" FontWeight="SemiBold" FontSize="14"/>
</Expander.Header>
<Border Background="#0D0F12" BorderBrush="#2D3540" BorderThickness="1,0,1,1"
CornerRadius="0,0,6,6" Padding="16">
<StackPanel Spacing="16">
<!-- Destination Table -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Destination Table"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding DestinationTable}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="36"
FontFamily="JetBrains Mono"
Watermark="dbo.WorkOrder_Curr"/>
<TextBlock Text="Target table in SQL Server (include schema)"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Match Columns -->
<StackPanel Spacing="4">
<StackPanel Orientation="Horizontal" Spacing="2">
<TextBlock Text="Match Columns (one per line)"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding MatchColumnsText}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="12"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="60"
Watermark="OrderNumber&#x0a;OrderType"/>
<TextBlock Text="Columns used to match source rows to existing destination rows"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Exclude From Update -->
<StackPanel Spacing="4">
<TextBlock Text="Exclude From Update (one per line)"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBox Text="{Binding ExcludeFromUpdateText}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="12"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="60"
Watermark="CreatedDate&#x0a;CreatedBy"/>
<TextBlock Text="Columns to skip when updating existing rows"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
<!-- Post Scripts -->
<StackPanel Spacing="4">
<TextBlock Text="Post Scripts (one per line)"
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBox Text="{Binding PostScriptsText}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550"
FontFamily="JetBrains Mono" FontSize="12"
AcceptsReturn="True"
TextWrapping="NoWrap"
MinHeight="60"
Watermark="EXEC dbo.UpdateStats @TableName"/>
<TextBlock Text="SQL scripts to execute after sync completes"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
</StackPanel>
</Border>
</Expander>
</StackPanel>
</ScrollViewer>
</UserControl>
@@ -1,11 +0,0 @@
using Avalonia.Controls;
namespace JdeScoping.ConfigManager.Views.Forms;
public partial class PipelineFormView : UserControl
{
public PipelineFormView()
{
InitializeComponent();
}
}