Joseph Doherty
604bfe919c
Apply comprehensive fixes from code reviews including: - Extract shared utilities (SqlFormatHelper, CellValueConverter, DbDestinationBase) - Add interface abstractions (IAuthenticationService, IDatabaseMigrator, IMisQueryBuilder) - Implement SecureStore for encrypted secrets storage - Fix error handling with proper HTTP status codes and logging - Optimize double enumeration in DevEtlRegistry - Add DataSync.Dev README for developer onboarding - Extract filter panel base classes to reduce duplication - Update code review docs to mark all issues as fixed
109 lines
4.1 KiB
C#
109 lines
4.1 KiB
C#
using JdeScoping.Core.Models.Enums;
|
|
using JdeScoping.Core.Models.Pipelines;
|
|
using JdeScoping.DataSync.Configuration;
|
|
|
|
namespace JdeScoping.Api.Mapping;
|
|
|
|
/// <summary>
|
|
/// Maps pipeline configuration to DTOs.
|
|
/// </summary>
|
|
public class PipelineMapper : IPipelineMapper
|
|
{
|
|
/// <inheritdoc />
|
|
public PipelineConfigDto MapToDto(
|
|
string name,
|
|
PipelineConfig config,
|
|
ScheduleDefaults defaults)
|
|
{
|
|
var source = new PipelineSourceDto(
|
|
config.Source.Connection,
|
|
Truncate(config.Source.Query),
|
|
Truncate(config.Source.MassQuery),
|
|
config.Source.Query,
|
|
config.Source.MassQuery,
|
|
config.Source.Parameters?.Select(p => new PipelineParameterDto(
|
|
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? []);
|
|
|
|
var matchCols = config.Destination.MatchColumns?.ToList();
|
|
var destination = new PipelineDestinationDto(
|
|
config.Destination.Table,
|
|
matchCols?.Count > 0 ? "BulkMerge" : "BulkImport",
|
|
matchCols,
|
|
config.Destination.ExcludeFromUpdate?.ToList());
|
|
|
|
// Mass uses massQuery with no parameters; Daily/Hourly use query with parameters
|
|
var parameters = config.Source.Parameters?.Select(p => new PipelineParameterDto(
|
|
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? [];
|
|
|
|
var schedules = new PipelineSchedulesDto(
|
|
MapSchedule(config.Schedules?.Mass, defaults.Mass, config.Source.MassQuery, [], config.PreScripts, config.PostScripts),
|
|
MapSchedule(config.Schedules?.Daily, defaults.Daily, config.Source.Query, parameters, config.PreScripts, config.PostScripts),
|
|
MapSchedule(config.Schedules?.Hourly, defaults.Hourly, config.Source.Query, parameters, config.PreScripts, config.PostScripts));
|
|
|
|
return new PipelineConfigDto(
|
|
name,
|
|
source,
|
|
destination,
|
|
schedules,
|
|
config.PreScripts?.Count ?? 0,
|
|
config.PostScripts?.Count ?? 0,
|
|
config.PreScripts,
|
|
config.PostScripts);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public ScheduleConfig? GetScheduleConfig(
|
|
PipelineConfig config,
|
|
UpdateTypes updateType) => updateType switch
|
|
{
|
|
UpdateTypes.Mass => config.Schedules?.Mass,
|
|
UpdateTypes.Daily => config.Schedules?.Daily,
|
|
UpdateTypes.Hourly => config.Schedules?.Hourly,
|
|
_ => null
|
|
};
|
|
|
|
/// <inheritdoc />
|
|
public int GetEffectiveInterval(
|
|
ScheduleConfig? config,
|
|
ScheduleDefaults defaults,
|
|
UpdateTypes updateType)
|
|
{
|
|
if (config?.IntervalMinutes > 0)
|
|
return config.IntervalMinutes;
|
|
|
|
return updateType switch
|
|
{
|
|
UpdateTypes.Mass => defaults.Mass.IntervalMinutes,
|
|
UpdateTypes.Daily => defaults.Daily.IntervalMinutes,
|
|
UpdateTypes.Hourly => defaults.Hourly.IntervalMinutes,
|
|
_ => 60
|
|
};
|
|
}
|
|
|
|
private static PipelineScheduleDto MapSchedule(
|
|
ScheduleConfig? config,
|
|
ScheduleConfig defaults,
|
|
string? query,
|
|
List<PipelineParameterDto> parameters,
|
|
List<string>? preScripts,
|
|
List<string>? postScripts)
|
|
{
|
|
return new PipelineScheduleDto(
|
|
config?.Enabled ?? defaults.Enabled,
|
|
config?.IntervalMinutes > 0 ? config.IntervalMinutes : defaults.IntervalMinutes,
|
|
config?.PrePurge ?? defaults.PrePurge,
|
|
config?.ReIndex ?? defaults.ReIndex,
|
|
config?.IntervalMinutes > 0 && config.IntervalMinutes != defaults.IntervalMinutes,
|
|
config?.PrePurge != null && config.PrePurge != defaults.PrePurge,
|
|
config?.ReIndex != null && config.ReIndex != defaults.ReIndex,
|
|
query,
|
|
parameters,
|
|
preScripts,
|
|
postScripts);
|
|
}
|
|
|
|
private static string? Truncate(string? value, int maxLength = 100) =>
|
|
value is null ? null :
|
|
value.Length <= maxLength ? value : value[..maxLength] + "...";
|
|
}
|