refactor: address code review findings across all projects

Apply comprehensive fixes from code reviews including: - Extract shared utilities (SqlFormatHelper, CellValueConverter, DbDestinationBase) - Add interface abstractions (IAuthenticationService, IDatabaseMigrator, IMisQueryBuilder) - Implement SecureStore for encrypted secrets storage - Fix error handling with proper HTTP status codes and logging - Optimize double enumeration in DevEtlRegistry - Add DataSync.Dev README for developer onboarding - Extract filter panel base classes to reduce duplication - Update code review docs to mark all issues as fixed
2026-01-19 11:05:36 -05:00
parent 08f5aa1447
commit 604bfe919c
148 changed files with 8696 additions and 1538 deletions
@@ -0,0 +1,108 @@
+using JdeScoping.Core.Models.Enums;
+using JdeScoping.Core.Models.Pipelines;
+using JdeScoping.DataSync.Configuration;
+
+namespace JdeScoping.Api.Mapping;
+
+/// <summary>
+/// Maps pipeline configuration to DTOs.
+/// </summary>
+public class PipelineMapper : IPipelineMapper
+{
+    /// <inheritdoc />
+    public PipelineConfigDto MapToDto(
+        string name,
+        PipelineConfig config,
+        ScheduleDefaults defaults)
+    {
+        var source = new PipelineSourceDto(
+            config.Source.Connection,
+            Truncate(config.Source.Query),
+            Truncate(config.Source.MassQuery),
+            config.Source.Query,
+            config.Source.MassQuery,
+            config.Source.Parameters?.Select(p => new PipelineParameterDto(
+                p.Key, p.Value.Format, p.Value.Source)).ToList() ?? []);
+
+        var matchCols = config.Destination.MatchColumns?.ToList();
+        var destination = new PipelineDestinationDto(
+            config.Destination.Table,
+            matchCols?.Count > 0 ? "BulkMerge" : "BulkImport",
+            matchCols,
+            config.Destination.ExcludeFromUpdate?.ToList());
+
+        // Mass uses massQuery with no parameters; Daily/Hourly use query with parameters
+        var parameters = config.Source.Parameters?.Select(p => new PipelineParameterDto(
+            p.Key, p.Value.Format, p.Value.Source)).ToList() ?? [];
+
+        var schedules = new PipelineSchedulesDto(
+            MapSchedule(config.Schedules?.Mass, defaults.Mass, config.Source.MassQuery, [], config.PreScripts, config.PostScripts),
+            MapSchedule(config.Schedules?.Daily, defaults.Daily, config.Source.Query, parameters, config.PreScripts, config.PostScripts),
+            MapSchedule(config.Schedules?.Hourly, defaults.Hourly, config.Source.Query, parameters, config.PreScripts, config.PostScripts));
+
+        return new PipelineConfigDto(
+            name,
+            source,
+            destination,
+            schedules,
+            config.PreScripts?.Count ?? 0,
+            config.PostScripts?.Count ?? 0,
+            config.PreScripts,
+            config.PostScripts);
+    }
+
+    /// <inheritdoc />
+    public ScheduleConfig? GetScheduleConfig(
+        PipelineConfig config,
+        UpdateTypes updateType) => updateType switch
+    {
+        UpdateTypes.Mass => config.Schedules?.Mass,
+        UpdateTypes.Daily => config.Schedules?.Daily,
+        UpdateTypes.Hourly => config.Schedules?.Hourly,
+        _ => null
+    };
+
+    /// <inheritdoc />
+    public int GetEffectiveInterval(
+        ScheduleConfig? config,
+        ScheduleDefaults defaults,
+        UpdateTypes updateType)
+    {
+        if (config?.IntervalMinutes > 0)
+            return config.IntervalMinutes;
+
+        return updateType switch
+        {
+            UpdateTypes.Mass => defaults.Mass.IntervalMinutes,
+            UpdateTypes.Daily => defaults.Daily.IntervalMinutes,
+            UpdateTypes.Hourly => defaults.Hourly.IntervalMinutes,
+            _ => 60
+        };
+    }
+
+    private static PipelineScheduleDto MapSchedule(
+        ScheduleConfig? config,
+        ScheduleConfig defaults,
+        string? query,
+        List<PipelineParameterDto> parameters,
+        List<string>? preScripts,
+        List<string>? postScripts)
+    {
+        return new PipelineScheduleDto(
+            config?.Enabled ?? defaults.Enabled,
+            config?.IntervalMinutes > 0 ? config.IntervalMinutes : defaults.IntervalMinutes,
+            config?.PrePurge ?? defaults.PrePurge,
+            config?.ReIndex ?? defaults.ReIndex,
+            config?.IntervalMinutes > 0 && config.IntervalMinutes != defaults.IntervalMinutes,
+            config?.PrePurge != null && config.PrePurge != defaults.PrePurge,
+            config?.ReIndex != null && config.ReIndex != defaults.ReIndex,
+            query,
+            parameters,
+            preScripts,
+            postScripts);
+    }
+
+    private static string? Truncate(string? value, int maxLength = 100) =>
+        value is null ? null :
+        value.Length <= maxLength ? value : value[..maxLength] + "...";
+}