dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: address code review findings across all projects

Browse Source 
Apply comprehensive fixes from code reviews including:
- Extract shared utilities (SqlFormatHelper, CellValueConverter, DbDestinationBase)
- Add interface abstractions (IAuthenticationService, IDatabaseMigrator, IMisQueryBuilder)
- Implement SecureStore for encrypted secrets storage
- Fix error handling with proper HTTP status codes and logging
- Optimize double enumeration in DevEtlRegistry
- Add DataSync.Dev README for developer onboarding
- Extract filter panel base classes to reduce duplication
- Update code review docs to mark all issues as fixed
This commit is contained in:
Joseph Doherty
2026-01-19 11:05:36 -05:00
parent 08f5aa1447
commit 604bfe919c
148 changed files with 8696 additions and 1538 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/JdeScoping.Api/Controllers/AuthController.cs
+3
View File
@@ -12,6 +12,9 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
// Alias to avoid conflict with Microsoft.AspNetCore.Authentication.IAuthenticationService
using IAuthService = JdeScoping.Core.Interfaces.IAuthenticationService;
namespace JdeScoping.Api.Controllers;
/// <summary>
NEW/src/JdeScoping.Api/Controllers/FileIOController.ComponentLots.cs
+4 -2
View File
@@ -16,13 +16,15 @@ public partial class FileIOController
/// </summary>
[HttpPost("componentlots/upload")]
[ProducesResponseType(typeof(FileUploadResult<LotViewModel>), StatusCodes.Status200OK)]
[ProducesResponseType(typeof(FileUploadResult<LotViewModel>), StatusCodes.Status400BadRequest)]
[ProducesResponseType(typeof(FileUploadResult<LotViewModel>), StatusCodes.Status422UnprocessableEntity)]
public async Task<ActionResult<FileUploadResult<LotViewModel>>> UploadComponentLots(
IFormFile? file,
CancellationToken ct)
{
if (file is null)
{
return Ok(new FileUploadResult<LotViewModel>
return BadRequest(new FileUploadResult<LotViewModel>
{
WasSuccessful = false,
ErrorMessage = "No file uploaded"
@@ -50,7 +52,7 @@ public partial class FileIOController
catch (Exception ex)
{
_logger.LogError(ex, "Failed to parse uploaded component lots file");
return Ok(new FileUploadResult<LotViewModel>
return UnprocessableEntity(new FileUploadResult<LotViewModel>
{
WasSuccessful = false,
ErrorMessage = "Failed to parse uploaded file"
NEW/src/JdeScoping.Api/Controllers/FileIOController.Items.cs
+4 -2
View File
@@ -16,13 +16,15 @@ public partial class FileIOController
/// </summary>
[HttpPost("items/upload")]
[ProducesResponseType(typeof(FileUploadResult<ItemViewModel>), StatusCodes.Status200OK)]
[ProducesResponseType(typeof(FileUploadResult<ItemViewModel>), StatusCodes.Status400BadRequest)]
[ProducesResponseType(typeof(FileUploadResult<ItemViewModel>), StatusCodes.Status422UnprocessableEntity)]
public async Task<ActionResult<FileUploadResult<ItemViewModel>>> UploadItems(
IFormFile? file,
CancellationToken ct)
{
if (file is null)
{
return Ok(new FileUploadResult<ItemViewModel>
return BadRequest(new FileUploadResult<ItemViewModel>
{
WasSuccessful = false,
ErrorMessage = "No file uploaded"
@@ -49,7 +51,7 @@ public partial class FileIOController
catch (Exception ex)
{
_logger.LogError(ex, "Failed to parse uploaded items file");
return Ok(new FileUploadResult<ItemViewModel>
return UnprocessableEntity(new FileUploadResult<ItemViewModel>
{
WasSuccessful = false,
ErrorMessage = "Failed to parse uploaded file"
NEW/src/JdeScoping.Api/Controllers/FileIOController.PartOperations.cs
+4 -2
View File
@@ -16,11 +16,13 @@ public partial class FileIOController
/// </summary>
[HttpPost("partoperations/upload")]
[ProducesResponseType(typeof(FileUploadResult<PartOperationViewModel>), StatusCodes.Status200OK)]
[ProducesResponseType(typeof(FileUploadResult<PartOperationViewModel>), StatusCodes.Status400BadRequest)]
[ProducesResponseType(typeof(FileUploadResult<PartOperationViewModel>), StatusCodes.Status422UnprocessableEntity)]
public ActionResult<FileUploadResult<PartOperationViewModel>> UploadPartOperations(IFormFile? file)
{
if (file is null)
{
return Ok(new FileUploadResult<PartOperationViewModel>
return BadRequest(new FileUploadResult<PartOperationViewModel>
{
WasSuccessful = false,
ErrorMessage = "No file uploaded"
@@ -41,7 +43,7 @@ public partial class FileIOController
catch (Exception ex)
{
_logger.LogError(ex, "Failed to parse uploaded part operations file");
return Ok(new FileUploadResult<PartOperationViewModel>
return UnprocessableEntity(new FileUploadResult<PartOperationViewModel>
{
WasSuccessful = false,
ErrorMessage = "Failed to parse uploaded file"
NEW/src/JdeScoping.Api/Controllers/FileIOController.WorkOrders.cs
+4 -2
View File
@@ -16,13 +16,15 @@ public partial class FileIOController
/// </summary>
[HttpPost("workorders/upload")]
[ProducesResponseType(typeof(FileUploadResult<WorkOrderViewModel>), StatusCodes.Status200OK)]
[ProducesResponseType(typeof(FileUploadResult<WorkOrderViewModel>), StatusCodes.Status400BadRequest)]
[ProducesResponseType(typeof(FileUploadResult<WorkOrderViewModel>), StatusCodes.Status422UnprocessableEntity)]
public async Task<ActionResult<FileUploadResult<WorkOrderViewModel>>> UploadWorkOrders(
IFormFile? file,
CancellationToken ct)
{
if (file is null)
{
return Ok(new FileUploadResult<WorkOrderViewModel>
return BadRequest(new FileUploadResult<WorkOrderViewModel>
{
WasSuccessful = false,
ErrorMessage = "No file uploaded"
@@ -50,7 +52,7 @@ public partial class FileIOController
catch (Exception ex)
{
_logger.LogError(ex, "Failed to parse uploaded work order file");
return Ok(new FileUploadResult<WorkOrderViewModel>
return UnprocessableEntity(new FileUploadResult<WorkOrderViewModel>
{
WasSuccessful = false,
ErrorMessage = "Failed to parse uploaded file"
NEW/src/JdeScoping.Api/Controllers/PipelineController.cs
+9 -101
View File
@@ -1,7 +1,7 @@
using JdeScoping.Api.Mapping;
using JdeScoping.Core.ApiContracts;
using JdeScoping.Core.Models.Pipelines;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Services;
using Microsoft.AspNetCore.Authorization;
@@ -12,20 +12,22 @@ namespace JdeScoping.Api.Controllers;
/// <summary>
/// API endpoints for pipeline configuration and status.
/// </summary>
[ApiController]
[Route(ApiRoutes.Pipelines.Base)]
[Authorize]
public class PipelineController : ControllerBase
public class PipelineController : ApiControllerBase
{
private readonly IEtlPipelineFactory _pipelineFactory;
private readonly IDataUpdateRepository _dataUpdateRepository;
private readonly IPipelineMapper _mapper;
public PipelineController(
IEtlPipelineFactory pipelineFactory,
IDataUpdateRepository dataUpdateRepository)
IDataUpdateRepository dataUpdateRepository,
IPipelineMapper mapper)
{
_pipelineFactory = pipelineFactory;
_dataUpdateRepository = dataUpdateRepository;
_mapper = mapper;
}
/// <summary>
@@ -51,7 +53,7 @@ public class PipelineController : ControllerBase
return NotFound();
var defaults = _pipelineFactory.GetScheduleDefaults();
var dto = MapToDto(name, config, defaults);
var dto = _mapper.MapToDto(name, config, defaults);
return Ok(dto);
}
@@ -75,8 +77,8 @@ public class PipelineController : ControllerBase
var statuses = new List<PipelineScheduleStatusDto>();
foreach (var updateType in new[] { UpdateTypes.Mass, UpdateTypes.Daily, UpdateTypes.Hourly })
{
var scheduleConfig = GetScheduleConfig(config, updateType);
var interval = GetEffectiveInterval(scheduleConfig, defaults, updateType);
var scheduleConfig = _mapper.GetScheduleConfig(config, updateType);
var interval = _mapper.GetEffectiveInterval(scheduleConfig, defaults, updateType);
lastRuns.TryGetValue(updateType, out var lastRun);
var successKey = $"{tableName}_{(int)updateType}";
@@ -127,98 +129,4 @@ public class PipelineController : ControllerBase
return Ok(new PipelineExecutionsResponse(executions));
}
private static PipelineConfigDto MapToDto(
string name,
PipelineConfig config,
ScheduleDefaults defaults)
{
var source = new PipelineSourceDto(
config.Source.Connection,
Truncate(config.Source.Query),
Truncate(config.Source.MassQuery),
config.Source.Query,
config.Source.MassQuery,
config.Source.Parameters?.Select(p => new PipelineParameterDto(
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? []);
var matchCols = config.Destination.MatchColumns?.ToList();
var destination = new PipelineDestinationDto(
config.Destination.Table,
matchCols?.Count > 0 ? "BulkMerge" : "BulkImport",
matchCols,
config.Destination.ExcludeFromUpdate?.ToList());
// Mass uses massQuery with no parameters; Daily/Hourly use query with parameters
var parameters = config.Source.Parameters?.Select(p => new PipelineParameterDto(
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? [];
var schedules = new PipelineSchedulesDto(
MapSchedule(config.Schedules?.Mass, defaults.Mass, config.Source.MassQuery, [], config.PreScripts, config.PostScripts),
MapSchedule(config.Schedules?.Daily, defaults.Daily, config.Source.Query, parameters, config.PreScripts, config.PostScripts),
MapSchedule(config.Schedules?.Hourly, defaults.Hourly, config.Source.Query, parameters, config.PreScripts, config.PostScripts));
return new PipelineConfigDto(
name,
source,
destination,
schedules,
config.PreScripts?.Count ?? 0,
config.PostScripts?.Count ?? 0,
config.PreScripts,
config.PostScripts);
}
private static PipelineScheduleDto MapSchedule(
ScheduleConfig? config,
ScheduleConfig defaults,
string? query,
List<PipelineParameterDto> parameters,
List<string>? preScripts,
List<string>? postScripts)
{
return new PipelineScheduleDto(
config?.Enabled ?? defaults.Enabled,
config?.IntervalMinutes > 0 ? config.IntervalMinutes : defaults.IntervalMinutes,
config?.PrePurge ?? defaults.PrePurge,
config?.ReIndex ?? defaults.ReIndex,
config?.IntervalMinutes > 0 && config.IntervalMinutes != defaults.IntervalMinutes,
config?.PrePurge != null && config.PrePurge != defaults.PrePurge,
config?.ReIndex != null && config.ReIndex != defaults.ReIndex,
query,
parameters,
preScripts,
postScripts);
}
private static ScheduleConfig? GetScheduleConfig(
PipelineConfig config,
UpdateTypes updateType) => updateType switch
{
UpdateTypes.Mass => config.Schedules?.Mass,
UpdateTypes.Daily => config.Schedules?.Daily,
UpdateTypes.Hourly => config.Schedules?.Hourly,
_ => null
};
private static int GetEffectiveInterval(
ScheduleConfig? config,
ScheduleDefaults defaults,
UpdateTypes updateType)
{
if (config?.IntervalMinutes > 0)
return config.IntervalMinutes;
return updateType switch
{
UpdateTypes.Mass => defaults.Mass.IntervalMinutes,
UpdateTypes.Daily => defaults.Daily.IntervalMinutes,
UpdateTypes.Hourly => defaults.Hourly.IntervalMinutes,
_ => 60
};
}
private static string? Truncate(string? value, int maxLength = 100) =>
value is null ? null :
value.Length <= maxLength ? value : value[..maxLength] + "...";
}
NEW/src/JdeScoping.Api/Controllers/SearchController.cs
+21 -6
View File
@@ -24,15 +24,18 @@ public class SearchController : ApiControllerBase
private readonly ILotFinderRepository _repository;
private readonly IHubContext<StatusHub> _hubContext;
private readonly ILogger<SearchController> _logger;
private readonly TimeProvider _timeProvider;
public SearchController(
ILotFinderRepository repository,
IHubContext<StatusHub> hubContext,
ILogger<SearchController> logger)
ILogger<SearchController> logger,
TimeProvider timeProvider)
{
_repository = repository;
_hubContext = hubContext;
_logger = logger;
_timeProvider = timeProvider;
}
/// <summary>
@@ -40,9 +43,13 @@ public class SearchController : ApiControllerBase
/// </summary>
[HttpGet]
[ProducesResponseType(typeof(IEnumerable<SearchViewModel>), StatusCodes.Status200OK)]
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
public async Task<ActionResult<IEnumerable<SearchViewModel>>> GetSearches(CancellationToken ct)
{
var searches = await _repository.GetUserSearchesAsync(CurrentUserName!, ct);
if (CurrentUserName is null)
return Unauthorized();
var searches = await _repository.GetUserSearchesAsync(CurrentUserName, ct);
var viewModels = searches
.OrderByDescending(s => s.StartDt)
.Select(s => new SearchViewModel(s));
@@ -82,9 +89,13 @@ public class SearchController : ApiControllerBase
/// </summary>
[HttpGet("{id:int}/copy")]
[ProducesResponseType(typeof(SearchViewModel), StatusCodes.Status200OK)]
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
[ProducesResponseType(StatusCodes.Status404NotFound)]
public async Task<ActionResult<SearchViewModel>> CopySearch(int id, CancellationToken ct)
{
if (CurrentUserName is null)
return Unauthorized();
var original = await _repository.GetSearchAsync(id, ct);
if (original is null)
{
@@ -95,7 +106,7 @@ public class SearchController : ApiControllerBase
var copy = new Search
{
Id = 0,
UserName = CurrentUserName!,
UserName = CurrentUserName,
Name = original.Name,
Status = SearchStatus.New,
SubmitDt = null,
@@ -112,12 +123,16 @@ public class SearchController : ApiControllerBase
/// </summary>
[HttpPost]
[ProducesResponseType(typeof(int), StatusCodes.Status201Created)]
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
public async Task<ActionResult<int>> CreateSearch(
[FromBody] SearchViewModel viewModel,
CancellationToken ct)
{
if (CurrentUserName is null)
return Unauthorized();
var search = viewModel.ToEntity();
search.UserName = CurrentUserName!;
search.UserName = CurrentUserName;
var searchId = await _repository.SubmitSearchAsync(search, ct);
@@ -127,10 +142,10 @@ public class SearchController : ApiControllerBase
var searchUpdate = new SearchUpdate
{
Id = searchId,
UserName = CurrentUserName!,
UserName = CurrentUserName,
Name = search.Name,
Status = search.Status,
Timestamp = DateTime.UtcNow
Timestamp = _timeProvider.GetUtcNow().DateTime
};
await _hubContext.Clients.All.SendAsync("searchUpdate", searchUpdate, ct);
}
NEW/src/JdeScoping.Api/DependencyInjection.cs
+7
View File
@@ -1,5 +1,6 @@
using System.Text.Json.Serialization;
using JdeScoping.Api.Hubs;
using JdeScoping.Api.Mapping;
using JdeScoping.Api.Options;
using JdeScoping.Api.Services;
using JdeScoping.Core.Interfaces;
@@ -35,6 +36,12 @@ public static class ApiDependencyInjection
// Register memory cache for file downloads
services.AddMemoryCache();
// Register TimeProvider for testability (allows mocking DateTime.UtcNow)
services.AddSingleton(TimeProvider.System);
// Register mappers
services.AddSingleton<IPipelineMapper, PipelineMapper>();
// Configure SignalR
services.AddSignalR();
NEW/src/JdeScoping.Api/Hubs/StatusHub.cs
+13 -8
View File
@@ -1,5 +1,6 @@
using JdeScoping.Core.ViewModels;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
namespace JdeScoping.Api.Hubs;
@@ -9,17 +10,17 @@ namespace JdeScoping.Api.Hubs;
/// </summary>
public class StatusHub : Hub
{
private static StatusUpdateViewModel _cachedStatus = new()
{
Message = "Unknown",
Timestamp = DateTime.UtcNow
};
private const string StatusCacheKey = "StatusHub_CachedStatus";
private readonly IMemoryCache _cache;
private readonly ILogger<StatusHub> _logger;
private readonly TimeProvider _timeProvider;
public StatusHub(ILogger<StatusHub> logger)
public StatusHub(IMemoryCache cache, ILogger<StatusHub> logger, TimeProvider timeProvider)
{
_cache = cache;
_logger = logger;
_timeProvider = timeProvider;
}
/// <summary>
@@ -29,7 +30,7 @@ public class StatusHub : Hub
/// <param name="statusUpdate">Status update to broadcast</param>
public async Task SetStatus(StatusUpdateViewModel statusUpdate)
{
_cachedStatus = statusUpdate;
_cache.Set(StatusCacheKey, statusUpdate);
await Clients.All.SendAsync("statusUpdate", statusUpdate);
_logger.LogDebug("Status updated: {Message}", statusUpdate.Message);
}
@@ -40,7 +41,11 @@ public class StatusHub : Hub
/// <returns>The most recent status update</returns>
public StatusUpdateViewModel GetCachedStatus()
{
return _cachedStatus;
return _cache.GetOrCreate(StatusCacheKey, _ => new StatusUpdateViewModel
{
Message = "Unknown",
Timestamp = _timeProvider.GetUtcNow().DateTime
})!;
}
/// <summary>
NEW/src/JdeScoping.Api/Mapping/IPipelineMapper.cs
+26
View File
@@ -0,0 +1,26 @@
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Pipelines;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.Api.Mapping;
/// <summary>
/// Mapper for pipeline configuration to DTOs.
/// </summary>
public interface IPipelineMapper
{
/// <summary>
/// Maps a pipeline configuration to its DTO representation.
/// </summary>
PipelineConfigDto MapToDto(string name, PipelineConfig config, ScheduleDefaults defaults);
/// <summary>
/// Gets the effective interval for a schedule, applying defaults if not specified.
/// </summary>
int GetEffectiveInterval(ScheduleConfig? config, ScheduleDefaults defaults, UpdateTypes updateType);
/// <summary>
/// Gets the schedule configuration for a specific update type.
/// </summary>
ScheduleConfig? GetScheduleConfig(PipelineConfig config, UpdateTypes updateType);
}
NEW/src/JdeScoping.Api/Mapping/PipelineMapper.cs
+108
View File
@@ -0,0 +1,108 @@
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Pipelines;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.Api.Mapping;
/// <summary>
/// Maps pipeline configuration to DTOs.
/// </summary>
public class PipelineMapper : IPipelineMapper
{
/// <inheritdoc />
public PipelineConfigDto MapToDto(
string name,
PipelineConfig config,
ScheduleDefaults defaults)
{
var source = new PipelineSourceDto(
config.Source.Connection,
Truncate(config.Source.Query),
Truncate(config.Source.MassQuery),
config.Source.Query,
config.Source.MassQuery,
config.Source.Parameters?.Select(p => new PipelineParameterDto(
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? []);
var matchCols = config.Destination.MatchColumns?.ToList();
var destination = new PipelineDestinationDto(
config.Destination.Table,
matchCols?.Count > 0 ? "BulkMerge" : "BulkImport",
matchCols,
config.Destination.ExcludeFromUpdate?.ToList());
// Mass uses massQuery with no parameters; Daily/Hourly use query with parameters
var parameters = config.Source.Parameters?.Select(p => new PipelineParameterDto(
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? [];
var schedules = new PipelineSchedulesDto(
MapSchedule(config.Schedules?.Mass, defaults.Mass, config.Source.MassQuery, [], config.PreScripts, config.PostScripts),
MapSchedule(config.Schedules?.Daily, defaults.Daily, config.Source.Query, parameters, config.PreScripts, config.PostScripts),
MapSchedule(config.Schedules?.Hourly, defaults.Hourly, config.Source.Query, parameters, config.PreScripts, config.PostScripts));
return new PipelineConfigDto(
name,
source,
destination,
schedules,
config.PreScripts?.Count ?? 0,
config.PostScripts?.Count ?? 0,
config.PreScripts,
config.PostScripts);
}
/// <inheritdoc />
public ScheduleConfig? GetScheduleConfig(
PipelineConfig config,
UpdateTypes updateType) => updateType switch
{
UpdateTypes.Mass => config.Schedules?.Mass,
UpdateTypes.Daily => config.Schedules?.Daily,
UpdateTypes.Hourly => config.Schedules?.Hourly,
_ => null
};
/// <inheritdoc />
public int GetEffectiveInterval(
ScheduleConfig? config,
ScheduleDefaults defaults,
UpdateTypes updateType)
{
if (config?.IntervalMinutes > 0)
return config.IntervalMinutes;
return updateType switch
{
UpdateTypes.Mass => defaults.Mass.IntervalMinutes,
UpdateTypes.Daily => defaults.Daily.IntervalMinutes,
UpdateTypes.Hourly => defaults.Hourly.IntervalMinutes,
_ => 60
};
}
private static PipelineScheduleDto MapSchedule(
ScheduleConfig? config,
ScheduleConfig defaults,
string? query,
List<PipelineParameterDto> parameters,
List<string>? preScripts,
List<string>? postScripts)
{
return new PipelineScheduleDto(
config?.Enabled ?? defaults.Enabled,
config?.IntervalMinutes > 0 ? config.IntervalMinutes : defaults.IntervalMinutes,
config?.PrePurge ?? defaults.PrePurge,
config?.ReIndex ?? defaults.ReIndex,
config?.IntervalMinutes > 0 && config.IntervalMinutes != defaults.IntervalMinutes,
config?.PrePurge != null && config.PrePurge != defaults.PrePurge,
config?.ReIndex != null && config.ReIndex != defaults.ReIndex,
query,
parameters,
preScripts,
postScripts);
}
private static string? Truncate(string? value, int maxLength = 100) =>
value is null ? null :
value.Length <= maxLength ? value : value[..maxLength] + "...";
}