feat: implement ETL pipeline redesign and ConfigManager improvements

- Add pipeline registry with JSON-based configuration and hot-reload support
- Implement manual sync request feature with API, client UI, and database
- Improve ConfigManager: connection string dropdown in pipeline editor,
  step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
This commit is contained in:
Joseph Doherty
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
@@ -1,8 +1,12 @@
using System.Text.Json.Serialization;
namespace JdeScoping.ConfigManager.Models;
/// <summary>
/// Configuration section for connection strings.
/// Supports both the standard .NET dictionary format and structured entries list.
/// </summary>
[JsonConverter(typeof(ConnectionStringsSectionConverter))]
public class ConnectionStringsSection
{
public List<ConnectionStringEntry> Entries { get; set; } = new();
@@ -0,0 +1,293 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace JdeScoping.ConfigManager.Models;
/// <summary>
/// Custom JSON converter that handles the standard .NET ConnectionStrings dictionary format
/// and converts it to a ConnectionStringsSection with an Entries list.
/// </summary>
/// <remarks>
/// Standard appsettings.json uses a dictionary format:
/// <code>
/// "ConnectionStrings": {
/// "LotFinder": "Server=localhost;Database=...;",
/// "JDE": "Data Source=...;"
/// }
/// </code>
/// This converter parses that into a list of ConnectionStringEntry objects.
/// </remarks>
public class ConnectionStringsSectionConverter : JsonConverter<ConnectionStringsSection>
{
public override ConnectionStringsSection? Read(
ref Utf8JsonReader reader,
Type typeToConvert,
JsonSerializerOptions options)
{
var section = new ConnectionStringsSection();
if (reader.TokenType != JsonTokenType.StartObject)
{
return section;
}
while (reader.Read())
{
if (reader.TokenType == JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType == JsonTokenType.PropertyName)
{
var propertyName = reader.GetString();
reader.Read();
// Check if this is the "Entries" property (our internal format)
if (string.Equals(propertyName, "entries", StringComparison.OrdinalIgnoreCase))
{
// Read as array of ConnectionStringEntry
if (reader.TokenType == JsonTokenType.StartArray)
{
section.Entries = JsonSerializer.Deserialize<List<ConnectionStringEntry>>(ref reader, options)
?? new List<ConnectionStringEntry>();
}
}
else if (!string.IsNullOrEmpty(propertyName))
{
// Standard dictionary format: property name is the connection name,
// value is the connection string
var connectionString = reader.TokenType == JsonTokenType.String
? reader.GetString()
: null;
if (!string.IsNullOrEmpty(connectionString))
{
var entry = ParseConnectionString(propertyName, connectionString);
section.Entries.Add(entry);
}
}
}
}
return section;
}
public override void Write(
Utf8JsonWriter writer,
ConnectionStringsSection value,
JsonSerializerOptions options)
{
// Write back in standard dictionary format for compatibility
writer.WriteStartObject();
foreach (var entry in value.Entries)
{
if (!string.IsNullOrEmpty(entry.Name))
{
writer.WriteString(entry.Name, entry.GenerateConnectionString());
}
}
writer.WriteEndObject();
}
/// <summary>
/// Parses a connection string and attempts to detect the provider type.
/// </summary>
private static ConnectionStringEntry ParseConnectionString(string name, string connectionString)
{
var entry = new ConnectionStringEntry
{
Name = name,
RawConnectionString = connectionString
};
// Try to detect provider and parse structured fields
var parts = ParseConnectionStringParts(connectionString);
// Detect Oracle first (Data Source with host:port/service pattern)
if (parts.TryGetValue("data source", out var dataSource) &&
IsOracleDataSource(dataSource))
{
entry.Provider = ConnectionProvider.Oracle;
ParseOracleDataSource(entry, dataSource);
if (parts.TryGetValue("user id", out var oraUserId))
{
entry.UserId = oraUserId;
}
if (parts.TryGetValue("password", out var oraPassword))
{
entry.Password = oraPassword;
}
}
// Detect SQL Server (Server property or Data Source without Oracle pattern)
else if (parts.TryGetValue("server", out var server) ||
parts.TryGetValue("data source", out server))
{
entry.Provider = ConnectionProvider.SqlServer;
entry.Server = server;
if (parts.TryGetValue("database", out var database) ||
parts.TryGetValue("initial catalog", out database))
{
entry.Database = database;
}
if (parts.TryGetValue("user id", out var userId) ||
parts.TryGetValue("uid", out userId))
{
entry.UserId = userId;
}
if (parts.TryGetValue("password", out var password) ||
parts.TryGetValue("pwd", out password))
{
entry.Password = password;
}
if (parts.TryGetValue("encrypt", out var encrypt))
{
entry.Encrypt = encrypt;
}
if (parts.TryGetValue("trustservercertificate", out var trustCert) ||
parts.TryGetValue("trust server certificate", out trustCert))
{
entry.TrustServerCertificate = trustCert.Equals("true", StringComparison.OrdinalIgnoreCase);
}
if (parts.TryGetValue("connection timeout", out var timeout) ||
parts.TryGetValue("connect timeout", out timeout))
{
if (int.TryParse(timeout, out var timeoutValue))
{
entry.ConnectionTimeout = timeoutValue;
}
}
if (parts.TryGetValue("application name", out var appName))
{
entry.ApplicationName = appName;
}
}
else
{
// Generic/unknown provider - just store raw connection string
entry.Provider = ConnectionProvider.Generic;
}
return entry;
}
/// <summary>
/// Parses a connection string into key-value pairs.
/// </summary>
private static Dictionary<string, string> ParseConnectionStringParts(string connectionString)
{
var parts = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
// Split by semicolon, handling quoted values
var segments = connectionString.Split(';', StringSplitOptions.RemoveEmptyEntries);
foreach (var segment in segments)
{
var equalsIndex = segment.IndexOf('=');
if (equalsIndex > 0)
{
var key = segment[..equalsIndex].Trim();
var value = segment[(equalsIndex + 1)..].Trim();
parts[key] = value;
}
}
return parts;
}
/// <summary>
/// Determines if a Data Source value looks like an Oracle connection string.
/// Oracle typically uses host:port/service format with numeric port and service name after slash.
/// SQL Server uses server,port or server\instance format.
/// </summary>
private static bool IsOracleDataSource(string dataSource)
{
// Check for Oracle patterns: //host:port/service or host:port/service
// SQL Server patterns: server,port or server\instance or just server
if (string.IsNullOrWhiteSpace(dataSource))
return false;
var source = dataSource.TrimStart('/');
// Oracle format: has colon followed by port number and then slash
// e.g., "jde-server:1521/JDEPROD" or "//db-host:1523/PRODDB"
var colonIndex = source.IndexOf(':');
var slashIndex = source.IndexOf('/');
if (colonIndex > 0 && slashIndex > colonIndex)
{
// Check if what's between colon and slash is a numeric port
var potentialPort = source[(colonIndex + 1)..slashIndex];
if (int.TryParse(potentialPort, out _))
{
return true;
}
}
// Also check for //host pattern without port
if (dataSource.StartsWith("//"))
{
return true;
}
return false;
}
/// <summary>
/// Parses Oracle Data Source format (e.g., "//host:port/service" or "host:port/service").
/// </summary>
private static void ParseOracleDataSource(ConnectionStringEntry entry, string dataSource)
{
// Remove leading slashes if present
var source = dataSource.TrimStart('/');
// Try to parse host:port/service format
var colonIndex = source.IndexOf(':');
var slashIndex = source.IndexOf('/');
if (colonIndex > 0)
{
entry.Host = source[..colonIndex];
if (slashIndex > colonIndex)
{
var portStr = source[(colonIndex + 1)..slashIndex];
if (int.TryParse(portStr, out var port))
{
entry.Port = port;
}
entry.ServiceName = source[(slashIndex + 1)..];
}
else
{
var portStr = source[(colonIndex + 1)..];
if (int.TryParse(portStr, out var port))
{
entry.Port = port;
}
}
}
else if (slashIndex > 0)
{
entry.Host = source[..slashIndex];
entry.ServiceName = source[(slashIndex + 1)..];
}
else
{
entry.Host = source;
}
}
}
@@ -1,4 +1,5 @@
using JdeScoping.ConfigManager.Models;
using JdeScoping.ConfigManager.Services;
using JdeScoping.ConfigManager.ViewModels.PipelineSteps;
using System.Collections.ObjectModel;
using System.Windows.Input;
@@ -12,14 +13,16 @@ public class PipelineEditorViewModel : ViewModelBase
{
private readonly PipelineModel _model;
private readonly Action _onChanged;
private readonly IDialogService _dialogService;
private PipelineStepViewModelBase? _selectedStep;
private object? _selectedStepEditor;
public PipelineEditorViewModel(string name, PipelineModel model, IReadOnlyList<string> availableConnections, Action onChanged)
public PipelineEditorViewModel(string name, PipelineModel model, IReadOnlyList<string> availableConnections, IDialogService dialogService, Action onChanged)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
_model = model ?? throw new ArgumentNullException(nameof(model));
AvailableConnections = availableConnections ?? [];
_dialogService = dialogService ?? throw new ArgumentNullException(nameof(dialogService));
_onChanged = onChanged ?? throw new ArgumentNullException(nameof(onChanged));
// Initialize collections
@@ -44,8 +47,11 @@ public class PipelineEditorViewModel : ViewModelBase
AddTransformerCommand = new RelayCommand(AddTransformer);
AddPostScriptCommand = new RelayCommand(AddPostScript);
RemoveStepCommand = new RelayCommand<PipelineStepViewModelBase>(RemoveStep);
DeleteSelectedStepCommand = new AsyncRelayCommand(DeleteSelectedStepAsync, () => CanDeleteSelectedStep);
MoveStepUpCommand = new RelayCommand<PipelineStepViewModelBase>(MoveStepUp, CanMoveStepUp);
MoveStepDownCommand = new RelayCommand<PipelineStepViewModelBase>(MoveStepDown, CanMoveStepDown);
MoveSelectedStepUpCommand = new RelayCommand(MoveSelectedStepUp, () => CanMoveSelectedStepUp);
MoveSelectedStepDownCommand = new RelayCommand(MoveSelectedStepDown, () => CanMoveSelectedStepDown);
}
/// <summary>
@@ -122,11 +128,35 @@ public class PipelineEditorViewModel : ViewModelBase
_selectedStep.IsSelected = true;
OnPropertyChanged();
OnPropertyChanged(nameof(CanDeleteSelectedStep));
OnPropertyChanged(nameof(CanMoveSelectedStepUp));
OnPropertyChanged(nameof(CanMoveSelectedStepDown));
(DeleteSelectedStepCommand as AsyncRelayCommand)?.RaiseCanExecuteChanged();
(MoveSelectedStepUpCommand as RelayCommand)?.RaiseCanExecuteChanged();
(MoveSelectedStepDownCommand as RelayCommand)?.RaiseCanExecuteChanged();
UpdateSelectedStepEditor();
}
}
}
/// <summary>
/// Gets a value indicating whether the selected step can be deleted.
/// Source and Destination steps cannot be deleted.
/// </summary>
public bool CanDeleteSelectedStep => _selectedStep is PreScriptStepViewModel
or TransformerStepViewModelBase
or PostScriptStepViewModel;
/// <summary>
/// Gets a value indicating whether the selected step can be moved up.
/// </summary>
public bool CanMoveSelectedStepUp => CanMoveStepUp(_selectedStep);
/// <summary>
/// Gets a value indicating whether the selected step can be moved down.
/// </summary>
public bool CanMoveSelectedStepDown => CanMoveStepDown(_selectedStep);
/// <summary>
/// Gets the editor view model for the currently selected step.
/// </summary>
@@ -156,8 +186,11 @@ public class PipelineEditorViewModel : ViewModelBase
public ICommand AddTransformerCommand { get; }
public ICommand AddPostScriptCommand { get; }
public ICommand RemoveStepCommand { get; }
public ICommand DeleteSelectedStepCommand { get; }
public ICommand MoveStepUpCommand { get; }
public ICommand MoveStepDownCommand { get; }
public ICommand MoveSelectedStepUpCommand { get; }
public ICommand MoveSelectedStepDownCommand { get; }
/// <summary>
/// Gets the list of available transformer types for the add dialog.
@@ -191,7 +224,7 @@ public class PipelineEditorViewModel : ViewModelBase
}
// Source
Source = new SourceStepViewModel(_model.Source, () =>
Source = new SourceStepViewModel(_model.Source, AvailableConnections, () =>
{
_onChanged();
});
@@ -411,6 +444,28 @@ public class PipelineEditorViewModel : ViewModelBase
OnPropertyChanged(nameof(AllSteps));
}
private void MoveSelectedStepUp()
{
if (_selectedStep == null) return;
MoveStepUp(_selectedStep);
RaiseMoveCanExecuteChanged();
}
private void MoveSelectedStepDown()
{
if (_selectedStep == null) return;
MoveStepDown(_selectedStep);
RaiseMoveCanExecuteChanged();
}
private void RaiseMoveCanExecuteChanged()
{
OnPropertyChanged(nameof(CanMoveSelectedStepUp));
OnPropertyChanged(nameof(CanMoveSelectedStepDown));
(MoveSelectedStepUpCommand as RelayCommand)?.RaiseCanExecuteChanged();
(MoveSelectedStepDownCommand as RelayCommand)?.RaiseCanExecuteChanged();
}
private static void MoveInCollection<T>(ObservableCollection<T> collection, T item, int offset)
{
var index = collection.IndexOf(item);
@@ -440,4 +495,31 @@ public class PipelineEditorViewModel : ViewModelBase
? PostScripts.Select(s => s.Script).ToArray()
: null;
}
/// <summary>
/// Deletes the currently selected step after user confirmation.
/// Source and Destination steps cannot be deleted.
/// </summary>
private async Task DeleteSelectedStepAsync()
{
if (_selectedStep == null || !CanDeleteSelectedStep)
return;
var stepTypeName = _selectedStep switch
{
PreScriptStepViewModel => "Pre-Script",
TransformerStepViewModelBase t => $"Transformer ({t.TransformerType})",
PostScriptStepViewModel => "Post-Script",
_ => "Step"
};
var confirmed = await _dialogService.ShowConfirmationAsync(
"Delete Step",
$"Are you sure you want to delete this {stepTypeName}?\n\nThis action cannot be undone.");
if (!confirmed)
return;
RemoveStep(_selectedStep);
}
}
@@ -573,9 +573,9 @@ public class MainWindowViewModel : ViewModelBase
MarkAsChanged,
_dialogService,
_connectionTestService),
_ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null
_ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null && _dialogService != null
=> _pipelines.Pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline)
? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), MarkAsChanged)
? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), _dialogService, MarkAsChanged)
: null,
_ => null
};
@@ -20,9 +20,10 @@ public class SourceStepViewModel : PipelineStepViewModelBase
{
private readonly PipelineSource _model;
public SourceStepViewModel(PipelineSource model, Action onChanged) : base(onChanged)
public SourceStepViewModel(PipelineSource model, IReadOnlyList<string> availableConnections, Action onChanged) : base(onChanged)
{
_model = model ?? throw new ArgumentNullException(nameof(model));
AvailableConnections = availableConnections ?? [];
// Initialize parameters collection
Parameters = new ObservableCollection<ParameterViewModel>(
@@ -36,6 +37,11 @@ public class SourceStepViewModel : PipelineStepViewModelBase
AddParameterCommand = new RelayCommand(AddParameter);
}
/// <summary>
/// Gets the list of available connection string names from configuration.
/// </summary>
public IReadOnlyList<string> AvailableConnections { get; }
public override PipelineStepType StepType => PipelineStepType.Source;
public override string DisplayName => "Source";
public override string Icon => IsFileSource ? "󰈔" : "󰆼"; // mdi-file vs mdi-database
@@ -39,12 +39,13 @@
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
</StackPanel>
<TextBox Text="{Binding Connection}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="36"
FontFamily="JetBrains Mono"
Watermark="jde"/>
<TextBlock Text="Connection string name (e.g., jde, cms, giw)"
<ComboBox ItemsSource="{Binding AvailableConnections}"
SelectedItem="{Binding Connection}"
Background="#232A35" Foreground="#E6EDF5"
BorderBrush="#3D4550" Height="36"
HorizontalAlignment="Stretch"
PlaceholderText="Select connection..."/>
<TextBlock Text="Connection string name from Settings > ConnectionStrings"
Foreground="#5C6A7A" FontSize="11"/>
</StackPanel>
@@ -279,40 +279,71 @@
</Border>
</StackPanel>
<!-- DataGrid -->
<DataGrid ItemsSource="{Binding Connections}"
SelectedItem="{Binding SelectedConnection}"
SelectionMode="Single"
Height="200"
Background="#0D0F12"
RowBackground="#0D0F12"
BorderBrush="#2D3540"
BorderThickness="1"
GridLinesVisibility="Horizontal"
HorizontalGridLinesBrush="#2D3540"
HeadersVisibility="Column"
AutoGenerateColumns="False"
CanUserReorderColumns="False"
CanUserResizeColumns="True"
CanUserSortColumns="True">
<DataGrid.Columns>
<DataGridTextColumn Header="Name"
Binding="{Binding Name}"
Width="*"
IsReadOnly="True"
Foreground="#E6EDF5"/>
<DataGridTextColumn Header="Provider"
Binding="{Binding ProviderDisplay}"
Width="100"
IsReadOnly="True"
Foreground="#9BA8B8"/>
<DataGridTextColumn Header="Server"
Binding="{Binding ServerDisplay}"
Width="150"
IsReadOnly="True"
Foreground="#9BA8B8"/>
</DataGrid.Columns>
</DataGrid>
<!-- Connection List -->
<Border BorderBrush="#2D3540" BorderThickness="1" CornerRadius="4">
<StackPanel>
<!-- Header Row -->
<Grid Background="#151920" Height="32">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="*"/>
<ColumnDefinition Width="100"/>
<ColumnDefinition Width="150"/>
</Grid.ColumnDefinitions>
<TextBlock Grid.Column="0" Text="Name" Foreground="#9BA8B8"
FontSize="12" FontWeight="Medium"
VerticalAlignment="Center" Margin="12,0"/>
<TextBlock Grid.Column="1" Text="Provider" Foreground="#9BA8B8"
FontSize="12" FontWeight="Medium"
VerticalAlignment="Center" Margin="8,0"/>
<TextBlock Grid.Column="2" Text="Server" Foreground="#9BA8B8"
FontSize="12" FontWeight="Medium"
VerticalAlignment="Center" Margin="8,0"/>
</Grid>
<!-- List Items -->
<ListBox ItemsSource="{Binding Connections}"
SelectedItem="{Binding SelectedConnection}"
SelectionMode="Single"
Background="#0D0F12"
MaxHeight="180"
MinHeight="100">
<ListBox.Styles>
<Style Selector="ListBoxItem">
<Setter Property="Padding" Value="0"/>
<Setter Property="Margin" Value="0"/>
<Setter Property="Background" Value="Transparent"/>
</Style>
<Style Selector="ListBoxItem:selected /template/ ContentPresenter">
<Setter Property="Background" Value="#2563EB"/>
</Style>
<Style Selector="ListBoxItem:pointerover /template/ ContentPresenter">
<Setter Property="Background" Value="#1E3A5F"/>
</Style>
</ListBox.Styles>
<ListBox.ItemTemplate>
<DataTemplate>
<Grid Height="36" Background="Transparent">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="*"/>
<ColumnDefinition Width="100"/>
<ColumnDefinition Width="150"/>
</Grid.ColumnDefinitions>
<TextBlock Grid.Column="0" Text="{Binding Name}"
Foreground="#E6EDF5" FontSize="13"
VerticalAlignment="Center" Margin="12,0"/>
<TextBlock Grid.Column="1" Text="{Binding ProviderDisplay}"
Foreground="#9BA8B8" FontSize="13"
VerticalAlignment="Center" Margin="8,0"/>
<TextBlock Grid.Column="2" Text="{Binding ServerDisplay}"
Foreground="#9BA8B8" FontSize="13"
VerticalAlignment="Center" Margin="8,0"/>
<Border Grid.ColumnSpan="3" BorderBrush="#2D3540"
BorderThickness="0,0,0,1" VerticalAlignment="Bottom"/>
</Grid>
</DataTemplate>
</ListBox.ItemTemplate>
</ListBox>
</StackPanel>
</Border>
<!-- Toolbar -->
<StackPanel Orientation="Horizontal" Spacing="8">
@@ -188,12 +188,49 @@
<Border Grid.Column="4" Background="#0D0F12" Padding="16">
<ScrollViewer VerticalScrollBarVisibility="Auto">
<StackPanel Spacing="16">
<!-- Properties Header -->
<StackPanel>
<TextBlock Text="PROPERTIES"
Foreground="#5C6A7A" FontSize="11" FontWeight="Medium"/>
<Border Height="1" Background="#2D3540" Margin="0,8,0,0"/>
</StackPanel>
<!-- Properties Header with Action Buttons -->
<Grid>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="*"/>
<ColumnDefinition Width="Auto"/>
</Grid.ColumnDefinitions>
<TextBlock Grid.Column="0" Text="PROPERTIES"
Foreground="#5C6A7A" FontSize="11" FontWeight="Medium"
VerticalAlignment="Center"/>
<StackPanel Grid.Column="1" Orientation="Horizontal" Spacing="4">
<!-- Move Up Button -->
<Button Command="{Binding MoveSelectedStepUpCommand}"
Background="#1F2937" Foreground="#9BA8B8"
BorderBrush="#3D4550" BorderThickness="1"
Height="28" Width="28" Padding="0"
ToolTip.Tip="Move step up"
IsVisible="{Binding CanDeleteSelectedStep}">
<TextBlock Text="▲" FontSize="10" HorizontalAlignment="Center"/>
</Button>
<!-- Move Down Button -->
<Button Command="{Binding MoveSelectedStepDownCommand}"
Background="#1F2937" Foreground="#9BA8B8"
BorderBrush="#3D4550" BorderThickness="1"
Height="28" Width="28" Padding="0"
ToolTip.Tip="Move step down"
IsVisible="{Binding CanDeleteSelectedStep}">
<TextBlock Text="▼" FontSize="10" HorizontalAlignment="Center"/>
</Button>
<!-- Delete Button -->
<Button Command="{Binding DeleteSelectedStepCommand}"
Background="#3D1F1F" Foreground="#FF6B6B"
BorderBrush="#5C2D2D" BorderThickness="1"
Height="28" Padding="12,0"
ToolTip.Tip="Delete selected step"
IsVisible="{Binding CanDeleteSelectedStep}">
<StackPanel Orientation="Horizontal" Spacing="6">
<TextBlock Text="🗑" FontSize="12"/>
<TextBlock Text="Delete" FontSize="11"/>
</StackPanel>
</Button>
</StackPanel>
</Grid>
<Border Height="1" Background="#2D3540" Margin="0,8,0,0"/>
<!-- Step Editor Content (changes based on selection) -->
<!-- Shows placeholder text when nothing selected, otherwise uses DataTemplates from MainWindow -->
@@ -36,6 +36,9 @@
<DataTemplate DataType="{x:Type forms:ExcelExportFormViewModel}">
<views:ExcelExportFormView/>
</DataTemplate>
<DataTemplate DataType="{x:Type forms:ConnectionStringsFormViewModel}">
<views:ConnectionStringsFormView/>
</DataTemplate>
<!-- Pipeline Editor (replaces PipelineFormViewModel) -->
<DataTemplate DataType="{x:Type forms:PipelineEditorViewModel}">