dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(configmanager): migrate to per-file pipeline system

Browse Source 
Align ConfigManager with DataSync's per-file pipeline format (pipeline.*.json)
by reusing EtlPipelineConfig types directly, eliminating duplicate models and
simplifying the codebase. Removes ~3200 lines of obsolete code.
This commit is contained in:
Joseph Doherty
2026-01-23 02:30:48 -05:00
parent 1b7bb26def
commit ba54a87be5
49 changed files with 1429 additions and 4396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/MainWindowViewModel.cs
+44 -25
View File
@@ -7,6 +7,7 @@ using JdeScoping.ConfigManager.Services;
using JdeScoping.ConfigManager.Services.SecureStore;
using JdeScoping.ConfigManager.ViewModels.Dialogs;
using JdeScoping.ConfigManager.ViewModels.Forms;
using JdeScoping.DataSync.Configuration;
using Microsoft.Extensions.Logging;
namespace JdeScoping.ConfigManager.ViewModels;
@@ -36,7 +37,7 @@ public class MainWindowViewModel : ViewModelBase
private object? _selectedFormViewModel;
private ConfigModel? _appSettings;
private PipelinesConfigModel? _pipelines;
private Dictionary<string, EtlPipelineConfig>? _pipelines;
/// <summary>
/// Gets or sets the currently loaded configuration folder path.
@@ -423,13 +424,17 @@ public class MainWindowViewModel : ViewModelBase
var appSettingsPath = Path.Combine(folderPath, "appsettings.json");
_appSettings = await _configFileService.LoadAppSettingsAsync(appSettingsPath);
// Use config-driven pipeline path
var pipelinesConfigPath = _appSettings?.Pipelines?.ConfigPath ?? "Pipelines/pipelines.json";
var pipelinesPath = Path.Combine(folderPath, pipelinesConfigPath);
// Load pipelines from directory containing pipeline.*.json files
var pipelinesDirectory = Path.Combine(folderPath,
_appSettings?.Pipelines?.ConfigDirectory ?? "Pipelines");
if (File.Exists(pipelinesPath))
if (Directory.Exists(pipelinesDirectory))
{
_pipelines = await _configFileService.LoadPipelinesAsync(pipelinesPath);
_pipelines = await _configFileService.LoadAllPipelinesAsync(pipelinesDirectory);
}
else
{
_pipelines = new Dictionary<string, EtlPipelineConfig>(StringComparer.OrdinalIgnoreCase);
}
// Initialize SecureStore (auto-create if needed, open, sync required keys)
@@ -468,7 +473,7 @@ public class MainWindowViewModel : ViewModelBase
var pipelinesFolder = new TreeNodeViewModel("Pipelines", "⚡", TreeNodeType.Folder) { IsExpanded = true };
if (_pipelines != null)
{
foreach (var (name, _) in _pipelines.Pipelines)
foreach (var name in _pipelines.Keys.OrderBy(k => k))
{
pipelinesFolder.Children.Add(new TreeNodeViewModel(name, "📦", TreeNodeType.Pipeline) { SectionKey = name });
}
@@ -574,7 +579,7 @@ public class MainWindowViewModel : ViewModelBase
_dialogService,
_connectionTestService),
_ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null && _dialogService != null
=> _pipelines.Pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline)
=> _pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline)
? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), _dialogService, MarkAsChanged)
: null,
_ => null
@@ -659,8 +664,8 @@ public class MainWindowViewModel : ViewModelBase
/// Loads configuration for testing purposes.
/// </summary>
/// <param name="appSettings">The application settings configuration model.</param>
/// <param name="pipelines">The pipelines configuration model.</param>
public void LoadConfigForTesting(ConfigModel? appSettings, PipelinesConfigModel? pipelines)
/// <param name="pipelines">The pipelines dictionary.</param>
public void LoadConfigForTesting(ConfigModel? appSettings, Dictionary<string, EtlPipelineConfig>? pipelines)
{
_appSettings = appSettings;
_pipelines = pipelines;
@@ -687,16 +692,19 @@ public class MainWindowViewModel : ViewModelBase
// Save appsettings
await _configFileService.SaveAppSettingsAsync(appSettingsPath, _appSettings);
// Save pipelines if loaded
// Save each pipeline to its own file
if (_pipelines != null)
{
var pipelinesConfigPath = _appSettings?.Pipelines?.ConfigPath ?? "Pipelines/pipelines.json";
var pipelinesPath = Path.Combine(ConfigFolderPath, pipelinesConfigPath);
if (File.Exists(pipelinesPath))
var pipelinesDirectory = Path.Combine(ConfigFolderPath,
_appSettings?.Pipelines?.ConfigDirectory ?? "Pipelines");
Directory.CreateDirectory(pipelinesDirectory);
foreach (var (name, pipeline) in _pipelines)
{
await _backupService.CreateBackupAsync(pipelinesPath);
var filePath = Path.Combine(pipelinesDirectory, $"pipeline.{name}.json");
await _configFileService.SavePipelineAsync(filePath, pipeline);
}
await _configFileService.SavePipelinesAsync(pipelinesPath, _pipelines);
}
HasUnsavedChanges = false;
@@ -834,22 +842,23 @@ public class MainWindowViewModel : ViewModelBase
return;
// Check for duplicate
if (_pipelines.Pipelines.ContainsKey(name))
if (_pipelines.ContainsKey(name))
{
await _dialogService.ShowMessageAsync("Error",
$"Pipeline '{name}' already exists.");
return;
}
// Create default pipeline model
var pipeline = new PipelineModel
// Create default pipeline using EtlPipelineConfig
var pipeline = new EtlPipelineConfig
{
Source = new PipelineSource { Connection = "lotfinder", Query = "" },
Destination = new PipelineDestination { Table = name },
Schedules = new PipelineSchedules()
Name = name,
IsEnabled = true,
Source = new SourceElement { Connection = "lotfinder", Query = "" },
Destination = new DestinationElement { Table = name }
};
_pipelines.Pipelines[name] = pipeline;
_pipelines[name] = pipeline;
// Add tree node
var pipelinesFolder = TreeNodes.FirstOrDefault(n =>
@@ -875,7 +884,8 @@ public class MainWindowViewModel : ViewModelBase
{
if (_selectedNode?.NodeType != TreeNodeType.Pipeline ||
_pipelines == null ||
_dialogService == null)
_dialogService == null ||
_appSettings == null)
return;
var name = _selectedNode.SectionKey!;
@@ -888,7 +898,16 @@ public class MainWindowViewModel : ViewModelBase
return;
// Remove from model
_pipelines.Pipelines.Remove(name);
_pipelines.Remove(name);
// Delete the pipeline file
var pipelinesDirectory = Path.Combine(ConfigFolderPath,
_appSettings.Pipelines?.ConfigDirectory ?? "Pipelines");
var filePath = Path.Combine(pipelinesDirectory, $"pipeline.{name}.json");
if (File.Exists(filePath))
{
await _configFileService.DeletePipelineFileAsync(filePath);
}
// Remove tree node
var pipelinesFolder = TreeNodes.FirstOrDefault(n =>