dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(webui): remove pipeline viewer feature

Browse Source 
Remove the read-only pipeline viewer from the web UI:
- Delete PipelineViewer.razor page and supporting components
- Delete PipelineController and PipelineMapper from API
- Delete Pipeline DTOs from Core
- Delete PipelineApiClient from Client
- Remove navigation link and DI registrations
- Delete obsolete plan documents

The ConfigManager utility retains pipeline editing capabilities.
This commit is contained in:
Joseph Doherty
2026-01-21 10:14:43 -05:00
parent 94d5a864e0
commit ceb63bfefb
20 changed files with 329 additions and 2776 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/JdeScoping.Api/Controllers/PipelineController.cs
-144
View File
@@ -1,144 +0,0 @@
using JdeScoping.Api.Mapping;
using JdeScoping.Core.ApiContracts;
using JdeScoping.Core.Models.Pipelines;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
namespace JdeScoping.Api.Controllers;
/// <summary>
/// API endpoints for pipeline configuration and status.
/// </summary>
[Route(ApiRoutes.Pipelines.Base)]
[Authorize]
public class PipelineController : ApiControllerBase
{
private readonly IEtlPipelineFactory _pipelineFactory;
private readonly IDataUpdateRepository _dataUpdateRepository;
private readonly IPipelineMapper _mapper;
/// <summary>
/// Initializes a new instance of the <see cref="PipelineController"/> class.
/// </summary>
/// <param name="pipelineFactory">The ETL pipeline factory.</param>
/// <param name="dataUpdateRepository">The data update repository.</param>
/// <param name="mapper">The pipeline mapper.</param>
public PipelineController(
IEtlPipelineFactory pipelineFactory,
IDataUpdateRepository dataUpdateRepository,
IPipelineMapper mapper)
{
_pipelineFactory = pipelineFactory;
_dataUpdateRepository = dataUpdateRepository;
_mapper = mapper;
}
/// <summary>
/// Gets list of all available pipeline names.
/// </summary>
[HttpGet]
public ActionResult<PipelineListResponse> GetPipelineNames()
{
var names = _pipelineFactory.GetAvailableTables()
.OrderBy(n => n)
.ToList();
return Ok(new PipelineListResponse(names));
}
/// <summary>
/// Gets configuration for a specific pipeline.
/// </summary>
/// <param name="name">The pipeline name.</param>
[HttpGet(ApiRoutes.Pipelines.ByName)]
public ActionResult<PipelineConfigDto> GetPipeline(string name)
{
var config = _pipelineFactory.GetPipelineConfig(name);
if (config is null)
return NotFound();
var defaults = _pipelineFactory.GetScheduleDefaults();
var dto = _mapper.MapToDto(name, config, defaults);
return Ok(dto);
}
/// <summary>
/// Gets schedule status for a pipeline.
/// </summary>
/// <param name="name">The pipeline name.</param>
/// <param name="cancellationToken">The cancellation token.</param>
[HttpGet(ApiRoutes.Pipelines.Status)]
public async Task<ActionResult<PipelineStatusResponse>> GetStatus(
string name,
CancellationToken cancellationToken)
{
var config = _pipelineFactory.GetPipelineConfig(name);
if (config is null)
return NotFound();
var tableName = config.Destination.Table;
var lastRuns = await _dataUpdateRepository.GetLastRunsAsync(tableName, cancellationToken);
var lastSuccessful = await _dataUpdateRepository.GetLastDataUpdatesAsync(cancellationToken);
var defaults = _pipelineFactory.GetScheduleDefaults();
var statuses = new List<PipelineScheduleStatusDto>();
foreach (var updateType in new[] { UpdateTypes.Mass, UpdateTypes.Daily, UpdateTypes.Hourly })
{
var scheduleConfig = _mapper.GetScheduleConfig(config, updateType);
var interval = _mapper.GetEffectiveInterval(scheduleConfig, defaults, updateType);
lastRuns.TryGetValue(updateType, out var lastRun);
var successKey = $"{tableName}_{(int)updateType}";
lastSuccessful.TryGetValue(successKey, out var lastSuccess);
var nextRequired = lastSuccess?.EndDt?.AddMinutes(interval);
var isOverdue = DataUpdateRepository.IsOverdue(
lastSuccess?.EndDt, tableName, updateType, null);
statuses.Add(new PipelineScheduleStatusDto(
updateType,
lastRun?.StartDt,
lastRun?.WasSuccessful ?? false,
lastSuccess?.EndDt,
nextRequired,
isOverdue,
interval));
}
return Ok(new PipelineStatusResponse(statuses));
}
/// <summary>
/// Gets recent execution history for a pipeline.
/// </summary>
/// <param name="name">The pipeline name.</param>
/// <param name="count">The maximum number of recent executions to retrieve.</param>
/// <param name="cancellationToken">The cancellation token.</param>
[HttpGet(ApiRoutes.Pipelines.Executions)]
public async Task<ActionResult<PipelineExecutionsResponse>> GetExecutions(
string name,
[FromQuery] int count = 30,
CancellationToken cancellationToken = default)
{
var config = _pipelineFactory.GetPipelineConfig(name);
if (config is null)
return NotFound();
var tableName = config.Destination.Table;
var updates = await _dataUpdateRepository.GetRecentUpdatesAsync(
tableName, null, count, cancellationToken);
var executions = updates.Select(u => new PipelineExecutionDto(
u.UpdateType,
u.StartDt,
u.EndDt == default ? null : u.EndDt,
u.EndDt == default ? null : u.EndDt - u.StartDt,
u.NumberRecords,
u.WasSuccessful
)).ToList();
return Ok(new PipelineExecutionsResponse(executions));
}
}
NEW/src/JdeScoping.Api/DependencyInjection.cs
-4
View File
@@ -1,6 +1,5 @@
using System.Text.Json.Serialization;
using JdeScoping.Api.Hubs;
using JdeScoping.Api.Mapping;
using JdeScoping.Api.Options;
using JdeScoping.Api.Services;
using JdeScoping.Core.Interfaces;
@@ -39,9 +38,6 @@ public static class ApiDependencyInjection
// Register TimeProvider for testability (allows mocking DateTime.UtcNow)
services.AddSingleton(TimeProvider.System);
// Register mappers
services.AddSingleton<IPipelineMapper, PipelineMapper>();
// Configure SignalR
services.AddSignalR();
NEW/src/JdeScoping.Api/Mapping/IPipelineMapper.cs
-34
View File
@@ -1,34 +0,0 @@
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Pipelines;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.Api.Mapping;
/// <summary>
/// Mapper for pipeline configuration to DTOs.
/// </summary>
public interface IPipelineMapper
{
/// <summary>
/// Maps a pipeline configuration to its DTO representation.
/// </summary>
/// <param name="name">The pipeline name.</param>
/// <param name="config">The pipeline configuration.</param>
/// <param name="defaults">The default schedule settings.</param>
PipelineConfigDto MapToDto(string name, PipelineConfig config, ScheduleDefaults defaults);
/// <summary>
/// Gets the effective interval for a schedule, applying defaults if not specified.
/// </summary>
/// <param name="config">The schedule configuration, or null to use defaults.</param>
/// <param name="defaults">The default schedule settings.</param>
/// <param name="updateType">The type of update to get the interval for.</param>
int GetEffectiveInterval(ScheduleConfig? config, ScheduleDefaults defaults, UpdateTypes updateType);
/// <summary>
/// Gets the schedule configuration for a specific update type.
/// </summary>
/// <param name="config">The pipeline configuration.</param>
/// <param name="updateType">The type of update to get the configuration for.</param>
ScheduleConfig? GetScheduleConfig(PipelineConfig config, UpdateTypes updateType);
}
NEW/src/JdeScoping.Api/Mapping/PipelineMapper.cs
-108
View File
@@ -1,108 +0,0 @@
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Pipelines;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.Api.Mapping;
/// <summary>
/// Maps pipeline configuration to DTOs.
/// </summary>
public class PipelineMapper : IPipelineMapper
{
/// <inheritdoc />
public PipelineConfigDto MapToDto(
string name,
PipelineConfig config,
ScheduleDefaults defaults)
{
var source = new PipelineSourceDto(
config.Source.Connection,
Truncate(config.Source.Query),
Truncate(config.Source.MassQuery),
config.Source.Query,
config.Source.MassQuery,
config.Source.Parameters?.Select(p => new PipelineParameterDto(
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? []);
var matchCols = config.Destination.MatchColumns?.ToList();
var destination = new PipelineDestinationDto(
config.Destination.Table,
matchCols?.Count > 0 ? "BulkMerge" : "BulkImport",
matchCols,
config.Destination.ExcludeFromUpdate?.ToList());
// Mass uses massQuery with no parameters; Daily/Hourly use query with parameters
var parameters = config.Source.Parameters?.Select(p => new PipelineParameterDto(
p.Key, p.Value.Format, p.Value.Source)).ToList() ?? [];
var schedules = new PipelineSchedulesDto(
MapSchedule(config.Schedules?.Mass, defaults.Mass, config.Source.MassQuery, [], config.PreScripts, config.PostScripts),
MapSchedule(config.Schedules?.Daily, defaults.Daily, config.Source.Query, parameters, config.PreScripts, config.PostScripts),
MapSchedule(config.Schedules?.Hourly, defaults.Hourly, config.Source.Query, parameters, config.PreScripts, config.PostScripts));
return new PipelineConfigDto(
name,
source,
destination,
schedules,
config.PreScripts?.Count ?? 0,
config.PostScripts?.Count ?? 0,
config.PreScripts,
config.PostScripts);
}
/// <inheritdoc />
public ScheduleConfig? GetScheduleConfig(
PipelineConfig config,
UpdateTypes updateType) => updateType switch
{
UpdateTypes.Mass => config.Schedules?.Mass,
UpdateTypes.Daily => config.Schedules?.Daily,
UpdateTypes.Hourly => config.Schedules?.Hourly,
_ => null
};
/// <inheritdoc />
public int GetEffectiveInterval(
ScheduleConfig? config,
ScheduleDefaults defaults,
UpdateTypes updateType)
{
if (config?.IntervalMinutes > 0)
return config.IntervalMinutes;
return updateType switch
{
UpdateTypes.Mass => defaults.Mass.IntervalMinutes,
UpdateTypes.Daily => defaults.Daily.IntervalMinutes,
UpdateTypes.Hourly => defaults.Hourly.IntervalMinutes,
_ => 60
};
}
private static PipelineScheduleDto MapSchedule(
ScheduleConfig? config,
ScheduleConfig defaults,
string? query,
List<PipelineParameterDto> parameters,
List<string>? preScripts,
List<string>? postScripts)
{
return new PipelineScheduleDto(
config?.Enabled ?? defaults.Enabled,
config?.IntervalMinutes > 0 ? config.IntervalMinutes : defaults.IntervalMinutes,
config?.PrePurge ?? defaults.PrePurge,
config?.ReIndex ?? defaults.ReIndex,
config?.IntervalMinutes > 0 && config.IntervalMinutes != defaults.IntervalMinutes,
config?.PrePurge != null && config.PrePurge != defaults.PrePurge,
config?.ReIndex != null && config.ReIndex != defaults.ReIndex,
query,
parameters,
preScripts,
postScripts);
}
private static string? Truncate(string? value, int maxLength = 100) =>
value is null ? null :
value.Length <= maxLength ? value : value[..maxLength] + "...";
}