dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: address code review findings across all projects

Browse Source 
Apply comprehensive fixes from code reviews including:
- Extract shared utilities (SqlFormatHelper, CellValueConverter, DbDestinationBase)
- Add interface abstractions (IAuthenticationService, IDatabaseMigrator, IMisQueryBuilder)
- Implement SecureStore for encrypted secrets storage
- Fix error handling with proper HTTP status codes and logging
- Optimize double enumeration in DevEtlRegistry
- Add DataSync.Dev README for developer onboarding
- Extract filter panel base classes to reduce duplication
- Update code review docs to mark all issues as fixed
This commit is contained in:
Joseph Doherty
2026-01-19 11:05:36 -05:00
parent 08f5aa1447
commit 604bfe919c
148 changed files with 8696 additions and 1538 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/JdeScoping.Api/Controllers/PipelineController.cs
+9 -101
View File
@@ -1,7 +1,7 @@
using JdeScoping.Api.Mapping;
using JdeScoping.Core.ApiContracts;
using JdeScoping.Core.Models.Pipelines;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Services;
using Microsoft.AspNetCore.Authorization;
@@ -12,20 +12,22 @@ namespace JdeScoping.Api.Controllers;
/// <summary>
/// API endpoints for pipeline configuration and status.
/// </summary>
[ApiController]
[Route(ApiRoutes.Pipelines.Base)]
[Authorize]
public class PipelineController : ControllerBase
public class PipelineController : ApiControllerBase
{
private readonly IEtlPipelineFactory _pipelineFactory;
private readonly IDataUpdateRepository _dataUpdateRepository;
private readonly IPipelineMapper _mapper;
public PipelineController(
IEtlPipelineFactory pipelineFactory,
IDataUpdateRepository dataUpdateRepository)
IDataUpdateRepository dataUpdateRepository,
IPipelineMapper mapper)
{
_pipelineFactory = pipelineFactory;
_dataUpdateRepository = dataUpdateRepository;
_mapper = mapper;
}
/// <summary>
@@ -51,7 +53,7 @@ public class PipelineController : ControllerBase
return NotFound();
var defaults = _pipelineFactory.GetScheduleDefaults();
var dto = MapToDto(name, config, defaults);
var dto = _mapper.MapToDto(name, config, defaults);
return Ok(dto);
}
@@ -75,8 +77,8 @@ public class PipelineController : ControllerBase
var statuses = new List<PipelineScheduleStatusDto>();
foreach (var updateType in new[] { UpdateTypes.Mass, UpdateTypes.Daily, UpdateTypes.Hourly })
{
var scheduleConfig = GetScheduleConfig(config, updateType);
var interval = GetEffectiveInterval(scheduleConfig, defaults, updateType);
var scheduleConfig = _mapper.GetScheduleConfig(config, updateType);
var interval = _mapper.GetEffectiveInterval(scheduleConfig, defaults, updateType);
lastRuns.TryGetValue(updateType, out var lastRun);
var successKey = $"{tableName}_{(int)updateType}";
@@ -127,98 +129,4 @@ public class PipelineController : ControllerBase
return Ok(new PipelineExecutionsResponse(executions));
}
private static PipelineConfigDto MapToDto(
string name,
PipelineConfig config,
ScheduleDefaults defaults)
{
var source = new PipelineSourceDto(
config.Source.Connection,
Truncate(config.Source.Query),
Truncate(config.Source.MassQuery),
config.Source.Query,
config.Source.MassQuery,
config.Source.Parameters?.Select(p => new PipelineParameterDto(
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? []);
var matchCols = config.Destination.MatchColumns?.ToList();
var destination = new PipelineDestinationDto(
config.Destination.Table,
matchCols?.Count > 0 ? "BulkMerge" : "BulkImport",
matchCols,
config.Destination.ExcludeFromUpdate?.ToList());
// Mass uses massQuery with no parameters; Daily/Hourly use query with parameters
var parameters = config.Source.Parameters?.Select(p => new PipelineParameterDto(
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? [];
var schedules = new PipelineSchedulesDto(
MapSchedule(config.Schedules?.Mass, defaults.Mass, config.Source.MassQuery, [], config.PreScripts, config.PostScripts),
MapSchedule(config.Schedules?.Daily, defaults.Daily, config.Source.Query, parameters, config.PreScripts, config.PostScripts),
MapSchedule(config.Schedules?.Hourly, defaults.Hourly, config.Source.Query, parameters, config.PreScripts, config.PostScripts));
return new PipelineConfigDto(
name,
source,
destination,
schedules,
config.PreScripts?.Count ?? 0,
config.PostScripts?.Count ?? 0,
config.PreScripts,
config.PostScripts);
}
private static PipelineScheduleDto MapSchedule(
ScheduleConfig? config,
ScheduleConfig defaults,
string? query,
List<PipelineParameterDto> parameters,
List<string>? preScripts,
List<string>? postScripts)
{
return new PipelineScheduleDto(
config?.Enabled ?? defaults.Enabled,
config?.IntervalMinutes > 0 ? config.IntervalMinutes : defaults.IntervalMinutes,
config?.PrePurge ?? defaults.PrePurge,
config?.ReIndex ?? defaults.ReIndex,
config?.IntervalMinutes > 0 && config.IntervalMinutes != defaults.IntervalMinutes,
config?.PrePurge != null && config.PrePurge != defaults.PrePurge,
config?.ReIndex != null && config.ReIndex != defaults.ReIndex,
query,
parameters,
preScripts,
postScripts);
}
private static ScheduleConfig? GetScheduleConfig(
PipelineConfig config,
UpdateTypes updateType) => updateType switch
{
UpdateTypes.Mass => config.Schedules?.Mass,
UpdateTypes.Daily => config.Schedules?.Daily,
UpdateTypes.Hourly => config.Schedules?.Hourly,
_ => null
};
private static int GetEffectiveInterval(
ScheduleConfig? config,
ScheduleDefaults defaults,
UpdateTypes updateType)
{
if (config?.IntervalMinutes > 0)
return config.IntervalMinutes;
return updateType switch
{
UpdateTypes.Mass => defaults.Mass.IntervalMinutes,
UpdateTypes.Daily => defaults.Daily.IntervalMinutes,
UpdateTypes.Hourly => defaults.Hourly.IntervalMinutes,
_ => 60
};
}
private static string? Truncate(string? value, int maxLength = 100) =>
value is null ? null :
value.Length <= maxLength ? value : value[..maxLength] + "...";
}