feat: implement ETL pipeline redesign and ConfigManager improvements
- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
namespace JdeScoping.Api.Contracts.ManualSync;
|
||||
|
||||
/// <summary>
|
||||
/// Request to cancel a manual sync request.
|
||||
/// </summary>
|
||||
public class CancelManualSyncRequestDto
|
||||
{
|
||||
/// <summary>
|
||||
/// The row version for optimistic concurrency (Base64 encoded).
|
||||
/// </summary>
|
||||
public string RowVersionBase64 { get; set; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace JdeScoping.Api.Contracts.ManualSync;
|
||||
|
||||
/// <summary>
|
||||
/// Request to create a new manual sync request.
|
||||
/// </summary>
|
||||
public class CreateManualSyncRequestDto
|
||||
{
|
||||
/// <summary>
|
||||
/// The name of the pipeline to sync.
|
||||
/// </summary>
|
||||
public string PipelineName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The type of sync to perform (mass, daily, hourly).
|
||||
/// </summary>
|
||||
public string SyncType { get; set; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
namespace JdeScoping.Api.Contracts.ManualSync;
|
||||
|
||||
/// <summary>
|
||||
/// View model for a manual sync request.
|
||||
/// </summary>
|
||||
public class ManualSyncRequestViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// The unique identifier for the sync request.
|
||||
/// </summary>
|
||||
public int Id { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The name of the pipeline being synced.
|
||||
/// </summary>
|
||||
public string PipelineName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The type of sync (mass, daily, hourly).
|
||||
/// </summary>
|
||||
public string SyncType { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The date and time the request was made.
|
||||
/// </summary>
|
||||
public DateTime RequestDT { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The username of the person who requested the sync.
|
||||
/// </summary>
|
||||
public string RequestedBy { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The date and time the sync completed, if applicable.
|
||||
/// </summary>
|
||||
public DateTime? CompletedDT { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The date and time the sync was cancelled, if applicable.
|
||||
/// </summary>
|
||||
public DateTime? CancelDT { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The username of the person who cancelled the sync, if applicable.
|
||||
/// </summary>
|
||||
public string? CancelledBy { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The current status of the sync request.
|
||||
/// </summary>
|
||||
public string Status { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The row version for optimistic concurrency (Base64 encoded).
|
||||
/// </summary>
|
||||
public string RowVersionBase64 { get; set; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace JdeScoping.Api.Contracts.ManualSync;
|
||||
|
||||
/// <summary>
|
||||
/// View model for pipeline information.
|
||||
/// </summary>
|
||||
public class PipelineInfoViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// The name of the pipeline.
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The sync types supported by this pipeline.
|
||||
/// </summary>
|
||||
public List<string> SupportedSyncTypes { get; set; } = new();
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
using JdeScoping.Api.Contracts.ManualSync;
|
||||
using JdeScoping.DataAccess.Services;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace JdeScoping.Api.Controllers;
|
||||
|
||||
/// <summary>
|
||||
/// API endpoints for manual data sync request management.
|
||||
/// </summary>
|
||||
[Route("api/manual-sync")]
|
||||
[ApiController]
|
||||
[Authorize]
|
||||
public class ManualSyncController : ApiControllerBase
|
||||
{
|
||||
private readonly IManualSyncRequestService _manualSyncRequestService;
|
||||
private readonly IPipelineRegistry _pipelineRegistry;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ManualSyncController"/> class.
|
||||
/// </summary>
|
||||
/// <param name="manualSyncRequestService">The service for managing manual sync requests.</param>
|
||||
/// <param name="pipelineRegistry">The pipeline registry for querying pipeline information.</param>
|
||||
public ManualSyncController(
|
||||
IManualSyncRequestService manualSyncRequestService,
|
||||
IPipelineRegistry pipelineRegistry)
|
||||
{
|
||||
_manualSyncRequestService = manualSyncRequestService;
|
||||
_pipelineRegistry = pipelineRegistry;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all manual sync requests, optionally filtered to pending only.
|
||||
/// </summary>
|
||||
/// <param name="pendingOnly">If true, returns only pending requests. Default is false.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A list of manual sync request view models.</returns>
|
||||
[HttpGet]
|
||||
[ProducesResponseType(typeof(List<ManualSyncRequestViewModel>), StatusCodes.Status200OK)]
|
||||
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
|
||||
public async Task<ActionResult<List<ManualSyncRequestViewModel>>> GetRequests(
|
||||
[FromQuery] bool pendingOnly = false,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var requests = await _manualSyncRequestService.GetRequestsAsync(pendingOnly, ct);
|
||||
|
||||
var viewModels = requests.Select(r => new ManualSyncRequestViewModel
|
||||
{
|
||||
Id = r.Id,
|
||||
PipelineName = r.PipelineName,
|
||||
SyncType = r.SyncType,
|
||||
RequestDT = r.RequestDT,
|
||||
RequestedBy = r.RequestedBy,
|
||||
CompletedDT = r.CompletedDT,
|
||||
CancelDT = r.CancelDT,
|
||||
CancelledBy = r.CancelledBy,
|
||||
Status = r.Status,
|
||||
RowVersionBase64 = Convert.ToBase64String(r.RowVersion)
|
||||
}).ToList();
|
||||
|
||||
return Ok(viewModels);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all available pipelines with their supported sync types.
|
||||
/// </summary>
|
||||
/// <returns>A list of pipeline information view models.</returns>
|
||||
[HttpGet("pipelines")]
|
||||
[ProducesResponseType(typeof(List<PipelineInfoViewModel>), StatusCodes.Status200OK)]
|
||||
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
|
||||
public ActionResult<List<PipelineInfoViewModel>> GetPipelines()
|
||||
{
|
||||
var pipelines = _pipelineRegistry.GetEnabledPipelines()
|
||||
.Select(p => new PipelineInfoViewModel
|
||||
{
|
||||
Name = p.Name,
|
||||
SupportedSyncTypes = GetSupportedSyncTypes(p)
|
||||
})
|
||||
.ToList();
|
||||
|
||||
return Ok(pipelines);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new manual sync request.
|
||||
/// </summary>
|
||||
/// <param name="dto">The create request containing pipeline name and sync type.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The created manual sync request view model.</returns>
|
||||
[HttpPost]
|
||||
[ProducesResponseType(typeof(ManualSyncRequestViewModel), StatusCodes.Status201Created)]
|
||||
[ProducesResponseType(StatusCodes.Status400BadRequest)]
|
||||
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
|
||||
public async Task<ActionResult<ManualSyncRequestViewModel>> CreateRequest(
|
||||
[FromBody] CreateManualSyncRequestDto dto,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var username = User.Identity?.Name;
|
||||
if (string.IsNullOrEmpty(username))
|
||||
{
|
||||
return Unauthorized();
|
||||
}
|
||||
|
||||
// Validate pipeline and sync type combination using the registry
|
||||
if (!_pipelineRegistry.IsValidPipelineAndSyncType(dto.PipelineName, dto.SyncType))
|
||||
{
|
||||
return BadRequest($"Invalid pipeline/sync type combination. Pipeline '{dto.PipelineName}' does not support sync type '{dto.SyncType}'.");
|
||||
}
|
||||
|
||||
var request = await _manualSyncRequestService.CreateRequestAsync(
|
||||
dto.PipelineName,
|
||||
dto.SyncType,
|
||||
username,
|
||||
ct);
|
||||
|
||||
var viewModel = new ManualSyncRequestViewModel
|
||||
{
|
||||
Id = request.Id,
|
||||
PipelineName = request.PipelineName,
|
||||
SyncType = request.SyncType,
|
||||
RequestDT = request.RequestDT,
|
||||
RequestedBy = request.RequestedBy,
|
||||
CompletedDT = request.CompletedDT,
|
||||
CancelDT = request.CancelDT,
|
||||
CancelledBy = request.CancelledBy,
|
||||
Status = request.Status,
|
||||
RowVersionBase64 = Convert.ToBase64String(request.RowVersion)
|
||||
};
|
||||
|
||||
return CreatedAtAction(nameof(GetRequests), viewModel);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancels a pending manual sync request.
|
||||
/// </summary>
|
||||
/// <param name="id">The ID of the request to cancel.</param>
|
||||
/// <param name="dto">The cancel request containing the row version for concurrency.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A success message if cancelled, or a conflict error if already processed.</returns>
|
||||
[HttpPost("{id:int}/cancel")]
|
||||
[ProducesResponseType(typeof(object), StatusCodes.Status200OK)]
|
||||
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
|
||||
[ProducesResponseType(StatusCodes.Status409Conflict)]
|
||||
public async Task<IActionResult> CancelRequest(
|
||||
int id,
|
||||
[FromBody] CancelManualSyncRequestDto dto,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var username = User.Identity?.Name;
|
||||
if (string.IsNullOrEmpty(username))
|
||||
{
|
||||
return Unauthorized();
|
||||
}
|
||||
|
||||
byte[] rowVersion;
|
||||
try
|
||||
{
|
||||
rowVersion = Convert.FromBase64String(dto.RowVersionBase64);
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
return BadRequest("Invalid RowVersionBase64 format.");
|
||||
}
|
||||
|
||||
var cancelled = await _manualSyncRequestService.CancelRequestAsync(
|
||||
id,
|
||||
username,
|
||||
rowVersion,
|
||||
ct);
|
||||
|
||||
if (!cancelled)
|
||||
{
|
||||
return Conflict(new { message = "The request has already been completed or cancelled, or the data has been modified by another user." });
|
||||
}
|
||||
|
||||
return Ok(new { message = "Request cancelled successfully." });
|
||||
}
|
||||
|
||||
private static List<string> GetSupportedSyncTypes(DataSync.Configuration.EtlPipelineConfig pipeline)
|
||||
{
|
||||
var types = new List<string>();
|
||||
if (pipeline.SupportsMassSync) types.Add("mass");
|
||||
if (pipeline.SupportsDailySync) types.Add("daily");
|
||||
if (pipeline.SupportsHourlySync) types.Add("hourly");
|
||||
return types;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
using JdeScoping.Api.Contracts.ManualSync;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace JdeScoping.Api.Controllers;
|
||||
|
||||
/// <summary>
|
||||
/// API endpoints for pipeline management and hot reload.
|
||||
/// </summary>
|
||||
[Route("api/pipelines")]
|
||||
[ApiController]
|
||||
[Authorize]
|
||||
public class PipelineController : ApiControllerBase
|
||||
{
|
||||
private readonly IPipelineRegistry _pipelineRegistry;
|
||||
private readonly ILogger<PipelineController> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PipelineController"/> class.
|
||||
/// </summary>
|
||||
/// <param name="pipelineRegistry">The pipeline registry.</param>
|
||||
/// <param name="logger">The logger.</param>
|
||||
public PipelineController(
|
||||
IPipelineRegistry pipelineRegistry,
|
||||
ILogger<PipelineController> logger)
|
||||
{
|
||||
_pipelineRegistry = pipelineRegistry;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all enabled pipelines with their supported sync types.
|
||||
/// </summary>
|
||||
/// <returns>A list of pipeline information view models.</returns>
|
||||
[HttpGet]
|
||||
[ProducesResponseType(typeof(List<PipelineInfoViewModel>), StatusCodes.Status200OK)]
|
||||
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
|
||||
public ActionResult<List<PipelineInfoViewModel>> GetPipelines()
|
||||
{
|
||||
var pipelines = _pipelineRegistry.GetEnabledPipelines()
|
||||
.Select(p => new PipelineInfoViewModel
|
||||
{
|
||||
Name = p.Name,
|
||||
SupportedSyncTypes = GetSupportedSyncTypes(p)
|
||||
})
|
||||
.ToList();
|
||||
|
||||
return Ok(pipelines);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets registry metadata including version and last load time.
|
||||
/// </summary>
|
||||
/// <returns>Registry metadata.</returns>
|
||||
[HttpGet("status")]
|
||||
[ProducesResponseType(typeof(PipelineRegistryStatusViewModel), StatusCodes.Status200OK)]
|
||||
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
|
||||
public ActionResult<PipelineRegistryStatusViewModel> GetStatus()
|
||||
{
|
||||
var allPipelines = _pipelineRegistry.GetAllPipelines();
|
||||
var enabledPipelines = _pipelineRegistry.GetEnabledPipelines();
|
||||
|
||||
return Ok(new PipelineRegistryStatusViewModel
|
||||
{
|
||||
Version = _pipelineRegistry.Version,
|
||||
LastLoadedAt = _pipelineRegistry.LastLoadedAt,
|
||||
TotalPipelines = allPipelines.Count,
|
||||
EnabledPipelines = enabledPipelines.Count
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reloads all pipeline definitions from disk.
|
||||
/// Requires Admin role.
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The reload result.</returns>
|
||||
[HttpPost("reload")]
|
||||
[Authorize(Roles = "Admin")]
|
||||
[ProducesResponseType(typeof(PipelineReloadResultViewModel), StatusCodes.Status200OK)]
|
||||
[ProducesResponseType(StatusCodes.Status401Unauthorized)]
|
||||
[ProducesResponseType(StatusCodes.Status403Forbidden)]
|
||||
[ProducesResponseType(StatusCodes.Status500InternalServerError)]
|
||||
public async Task<ActionResult<PipelineReloadResultViewModel>> ReloadPipelines(CancellationToken ct = default)
|
||||
{
|
||||
var username = User.Identity?.Name ?? "unknown";
|
||||
|
||||
_logger.LogInformation("Pipeline reload requested by {User}", username);
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _pipelineRegistry.ReloadAsync(ct);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Pipeline reload completed: Success={Success}, Loaded={Loaded}, Skipped={Skipped}, Version={Version}",
|
||||
result.Success,
|
||||
result.PipelinesLoaded,
|
||||
result.PipelinesSkipped,
|
||||
result.NewVersion);
|
||||
|
||||
return Ok(new PipelineReloadResultViewModel
|
||||
{
|
||||
Success = result.Success,
|
||||
PipelinesLoaded = result.PipelinesLoaded,
|
||||
PipelinesSkipped = result.PipelinesSkipped,
|
||||
PreviousVersion = result.PreviousVersion,
|
||||
NewVersion = result.NewVersion,
|
||||
Errors = result.Errors.Select(e => new PipelineLoadErrorViewModel
|
||||
{
|
||||
FileName = e.FileName,
|
||||
PipelineName = e.PipelineName,
|
||||
ErrorType = e.ErrorType,
|
||||
Messages = e.Messages
|
||||
}).ToList()
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Pipeline reload failed");
|
||||
return StatusCode(StatusCodes.Status500InternalServerError, new { message = "Pipeline reload failed. See server logs for details." });
|
||||
}
|
||||
}
|
||||
|
||||
private static List<string> GetSupportedSyncTypes(EtlPipelineConfig pipeline)
|
||||
{
|
||||
var types = new List<string>();
|
||||
if (pipeline.SupportsMassSync) types.Add("mass");
|
||||
if (pipeline.SupportsDailySync) types.Add("daily");
|
||||
if (pipeline.SupportsHourlySync) types.Add("hourly");
|
||||
return types;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// View model for pipeline registry status.
|
||||
/// </summary>
|
||||
public class PipelineRegistryStatusViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the current registry version.
|
||||
/// </summary>
|
||||
public int Version { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the timestamp of the last successful load.
|
||||
/// </summary>
|
||||
public DateTime? LastLoadedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the total number of pipelines.
|
||||
/// </summary>
|
||||
public int TotalPipelines { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of enabled pipelines.
|
||||
/// </summary>
|
||||
public int EnabledPipelines { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// View model for pipeline reload result.
|
||||
/// </summary>
|
||||
public class PipelineReloadResultViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the reload was successful.
|
||||
/// </summary>
|
||||
public bool Success { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of pipelines loaded.
|
||||
/// </summary>
|
||||
public int PipelinesLoaded { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of pipelines skipped.
|
||||
/// </summary>
|
||||
public int PipelinesSkipped { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the previous version.
|
||||
/// </summary>
|
||||
public int PreviousVersion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the new version.
|
||||
/// </summary>
|
||||
public int NewVersion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the list of errors.
|
||||
/// </summary>
|
||||
public List<PipelineLoadErrorViewModel> Errors { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// View model for pipeline load error.
|
||||
/// </summary>
|
||||
public class PipelineLoadErrorViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the file name.
|
||||
/// </summary>
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the pipeline name.
|
||||
/// </summary>
|
||||
public string PipelineName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the error type.
|
||||
/// </summary>
|
||||
public string ErrorType { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the error messages.
|
||||
/// </summary>
|
||||
public List<string> Messages { get; set; } = [];
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
@*
|
||||
NewSyncRequestDialog.razor - Dialog for creating new manual sync requests.
|
||||
|
||||
Allows users to select a pipeline and sync type to queue a new data sync request.
|
||||
The sync type dropdown is filtered based on the selected pipeline's supported types.
|
||||
*@
|
||||
@namespace JdeScoping.Client.Components.DataSync
|
||||
@using JdeScoping.Core.ViewModels
|
||||
|
||||
<RadzenStack Gap="1rem">
|
||||
<RadzenFormField Text="Pipeline" Style="width: 100%;">
|
||||
<RadzenDropDown @bind-Value="_selectedPipeline"
|
||||
Data="@Pipelines"
|
||||
TextProperty="Name"
|
||||
ValueProperty="Name"
|
||||
Placeholder="Select a pipeline..."
|
||||
Style="width: 100%;"
|
||||
Change="@OnPipelineChanged" />
|
||||
</RadzenFormField>
|
||||
|
||||
<RadzenFormField Text="Sync Type" Style="width: 100%;">
|
||||
<RadzenDropDown @bind-Value="_selectedSyncType"
|
||||
Data="@_availableSyncTypes"
|
||||
Placeholder="Select sync type..."
|
||||
Disabled="@(string.IsNullOrEmpty(_selectedPipeline))"
|
||||
Style="width: 100%;" />
|
||||
</RadzenFormField>
|
||||
|
||||
@if (!string.IsNullOrEmpty(_selectedPipeline) && !string.IsNullOrEmpty(_selectedSyncType))
|
||||
{
|
||||
<RadzenAlert AlertStyle="AlertStyle.Info" ShowIcon="true" Variant="Variant.Flat">
|
||||
<strong>@_selectedSyncType</strong> sync will be queued for <strong>@_selectedPipeline</strong>.
|
||||
</RadzenAlert>
|
||||
}
|
||||
|
||||
<RadzenStack Orientation="Orientation.Horizontal" JustifyContent="JustifyContent.End" Gap="0.5rem">
|
||||
<RadzenButton Text="Cancel"
|
||||
ButtonStyle="ButtonStyle.Light"
|
||||
Click="@OnCancelAsync" />
|
||||
<RadzenButton Text="Create Request"
|
||||
ButtonStyle="ButtonStyle.Primary"
|
||||
Click="@OnCreateAsync"
|
||||
Disabled="@(!CanCreate)"
|
||||
IsBusy="@_isCreating"
|
||||
BusyText="Creating..." />
|
||||
</RadzenStack>
|
||||
</RadzenStack>
|
||||
@@ -0,0 +1,114 @@
|
||||
using System.Net.Http.Json;
|
||||
using JdeScoping.Core.ViewModels;
|
||||
using Microsoft.AspNetCore.Components;
|
||||
using Radzen;
|
||||
|
||||
namespace JdeScoping.Client.Components.DataSync;
|
||||
|
||||
/// <summary>
|
||||
/// Dialog component for creating new manual sync requests.
|
||||
/// </summary>
|
||||
public partial class NewSyncRequestDialog : ComponentBase
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the Radzen dialog service for closing the dialog.
|
||||
/// </summary>
|
||||
[Inject]
|
||||
private DialogService DialogService { get; set; } = default!;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the HTTP client for API calls.
|
||||
/// </summary>
|
||||
[Inject]
|
||||
private HttpClient HttpClient { get; set; } = default!;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the available pipelines to select from.
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public List<PipelineInfoViewModel> Pipelines { get; set; } = new();
|
||||
|
||||
private string? _selectedPipeline;
|
||||
private string? _selectedSyncType;
|
||||
private List<string> _availableSyncTypes = new();
|
||||
private bool _isCreating;
|
||||
|
||||
/// <summary>
|
||||
/// Determines if the Create button should be enabled.
|
||||
/// </summary>
|
||||
private bool CanCreate => !string.IsNullOrEmpty(_selectedPipeline) &&
|
||||
!string.IsNullOrEmpty(_selectedSyncType) &&
|
||||
!_isCreating;
|
||||
|
||||
/// <summary>
|
||||
/// Handles pipeline selection change by filtering available sync types.
|
||||
/// </summary>
|
||||
private void OnPipelineChanged()
|
||||
{
|
||||
// Reset sync type when pipeline changes
|
||||
_selectedSyncType = null;
|
||||
|
||||
if (string.IsNullOrEmpty(_selectedPipeline))
|
||||
{
|
||||
_availableSyncTypes = new();
|
||||
return;
|
||||
}
|
||||
|
||||
// Find the selected pipeline and get its supported sync types
|
||||
var pipeline = Pipelines.FirstOrDefault(p => p.Name == _selectedPipeline);
|
||||
_availableSyncTypes = pipeline?.SupportedSyncTypes ?? new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles the Cancel button click by closing the dialog with null result.
|
||||
/// </summary>
|
||||
private void OnCancelAsync()
|
||||
{
|
||||
DialogService.Close(null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles the Create button click by calling the API and closing with the result.
|
||||
/// </summary>
|
||||
private async Task OnCreateAsync()
|
||||
{
|
||||
if (!CanCreate)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_isCreating = true;
|
||||
StateHasChanged();
|
||||
|
||||
try
|
||||
{
|
||||
var createDto = new CreateManualSyncRequestDto
|
||||
{
|
||||
PipelineName = _selectedPipeline!,
|
||||
SyncType = _selectedSyncType!
|
||||
};
|
||||
|
||||
var response = await HttpClient.PostAsJsonAsync("api/manual-sync", createDto);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var createdRequest = await response.Content.ReadFromJsonAsync<ManualSyncRequestViewModel>();
|
||||
DialogService.Close(createdRequest);
|
||||
}
|
||||
else
|
||||
{
|
||||
// On error, close with null (parent can show notification if needed)
|
||||
DialogService.Close(null);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// On exception, close with null
|
||||
DialogService.Close(null);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_isCreating = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@
|
||||
<NavLink class="nav-link" href="/search">New Search</NavLink>
|
||||
<NavLink class="nav-link" href="/search/queue">Search Queue</NavLink>
|
||||
<NavLink class="nav-link" href="/refresh-status">Refresh Status</NavLink>
|
||||
<NavLink class="nav-link" href="/data-sync/requests">Data Sync</NavLink>
|
||||
</nav>
|
||||
</div>
|
||||
<div class="navbar-right">
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
@*
|
||||
DataSyncRequests.razor - Main page for viewing and managing manual data sync requests.
|
||||
|
||||
Displays a list of manual sync requests with filtering, sorting, and pagination.
|
||||
Allows users to create new sync requests and cancel pending ones.
|
||||
*@
|
||||
@page "/data-sync/requests"
|
||||
@attribute [Authorize]
|
||||
@using JdeScoping.Core.ViewModels
|
||||
@using JdeScoping.Client.Components.DataSync
|
||||
|
||||
<PageTitle>Data Sync Requests - JDE Scoping Tool</PageTitle>
|
||||
|
||||
<!-- Header -->
|
||||
<RadzenStack Orientation="Orientation.Horizontal" AlignItems="AlignItems.Center" JustifyContent="JustifyContent.SpaceBetween" class="rz-mb-4">
|
||||
<RadzenText TextStyle="TextStyle.H4" class="rz-m-0">Data Sync Requests</RadzenText>
|
||||
<RadzenStack Orientation="Orientation.Horizontal" Gap="0.5rem">
|
||||
<RadzenButton Text="Reload Pipelines" Icon="sync" ButtonStyle="ButtonStyle.Secondary" Click="@ReloadPipelinesAsync" IsBusy="@_isReloadingPipelines" title="Reload pipeline definitions from disk (Admin only)" />
|
||||
<RadzenButton Text="New Request" Icon="add" ButtonStyle="ButtonStyle.Primary" Click="@OpenNewRequestDialogAsync" />
|
||||
</RadzenStack>
|
||||
</RadzenStack>
|
||||
|
||||
<!-- Filter Card -->
|
||||
<RadzenCard class="rz-mb-4">
|
||||
<RadzenStack Orientation="Orientation.Horizontal" AlignItems="AlignItems.Center" JustifyContent="JustifyContent.SpaceBetween">
|
||||
<RadzenStack Orientation="Orientation.Horizontal" AlignItems="AlignItems.Center" Gap="0.5rem">
|
||||
<RadzenCheckBox TValue="bool" Value="_showPendingOnly" Change="@OnFilterChanged" />
|
||||
<RadzenText TextStyle="TextStyle.Body1">Show pending only</RadzenText>
|
||||
</RadzenStack>
|
||||
<RadzenButton Text="Refresh" Icon="refresh" ButtonStyle="ButtonStyle.Light" Click="@LoadRequestsAsync" IsBusy="@_isLoading" />
|
||||
</RadzenStack>
|
||||
</RadzenCard>
|
||||
|
||||
@if (_isLoading)
|
||||
{
|
||||
<LoadingIndicator Message="Loading requests..." />
|
||||
}
|
||||
else if (!string.IsNullOrEmpty(_errorMessage))
|
||||
{
|
||||
<RadzenAlert AlertStyle="AlertStyle.Danger" ShowIcon="true" Variant="Variant.Flat" class="rz-mb-4">
|
||||
@_errorMessage
|
||||
</RadzenAlert>
|
||||
}
|
||||
else if (_requests.Count == 0)
|
||||
{
|
||||
<RadzenAlert AlertStyle="AlertStyle.Info" ShowIcon="true" Variant="Variant.Flat" class="rz-mb-4">
|
||||
No sync requests found. Click "New Request" to create one.
|
||||
</RadzenAlert>
|
||||
}
|
||||
else
|
||||
{
|
||||
<RadzenDataGrid @ref="_grid" Data="@_requests" TItem="ManualSyncRequestViewModel" AllowSorting="true" AllowPaging="true" PageSize="20"
|
||||
PagerHorizontalAlign="HorizontalAlign.Center" AllowColumnResize="true">
|
||||
<Columns>
|
||||
<RadzenDataGridColumn TItem="ManualSyncRequestViewModel" Property="PipelineName" Title="Pipeline" Width="150px" />
|
||||
<RadzenDataGridColumn TItem="ManualSyncRequestViewModel" Property="SyncType" Title="Type" Width="100px" />
|
||||
<RadzenDataGridColumn TItem="ManualSyncRequestViewModel" Property="RequestDT" Title="Requested" Width="160px">
|
||||
<Template Context="request">
|
||||
@request.RequestDT.ToString("MM/dd hh:mm")
|
||||
</Template>
|
||||
</RadzenDataGridColumn>
|
||||
<RadzenDataGridColumn TItem="ManualSyncRequestViewModel" Property="RequestedBy" Title="By" Width="120px" />
|
||||
<RadzenDataGridColumn TItem="ManualSyncRequestViewModel" Property="Status" Title="Status" Width="100px">
|
||||
<Template Context="request">
|
||||
<RadzenBadge BadgeStyle="@GetBadgeStyle(request.Status)" Text="@request.Status" />
|
||||
</Template>
|
||||
</RadzenDataGridColumn>
|
||||
<RadzenDataGridColumn TItem="ManualSyncRequestViewModel" Title="Actions" Width="100px" Sortable="false">
|
||||
<Template Context="request">
|
||||
@if (request.Status == "Pending")
|
||||
{
|
||||
<RadzenButton Text="Cancel" Size="ButtonSize.Small" ButtonStyle="ButtonStyle.Danger" Click="@(() => CancelRequestAsync(request))" />
|
||||
}
|
||||
</Template>
|
||||
</RadzenDataGridColumn>
|
||||
</Columns>
|
||||
</RadzenDataGrid>
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
using JdeScoping.Client.Components.DataSync;
|
||||
using JdeScoping.Core.ApiContracts;
|
||||
using JdeScoping.Core.ViewModels;
|
||||
using Microsoft.AspNetCore.Components;
|
||||
using Radzen;
|
||||
using Radzen.Blazor;
|
||||
|
||||
namespace JdeScoping.Client.Pages.DataSync;
|
||||
|
||||
/// <summary>
|
||||
/// Code-behind for the Data Sync Requests page.
|
||||
/// </summary>
|
||||
public partial class DataSyncRequests : ComponentBase
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the manual sync API client.
|
||||
/// </summary>
|
||||
[Inject]
|
||||
private IManualSyncApiClient ManualSyncApi { get; set; } = default!;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the pipeline API client.
|
||||
/// </summary>
|
||||
[Inject]
|
||||
private IPipelineApiClient PipelineApi { get; set; } = default!;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the Radzen dialog service.
|
||||
/// </summary>
|
||||
[Inject]
|
||||
private DialogService DialogService { get; set; } = default!;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the Radzen notification service.
|
||||
/// </summary>
|
||||
[Inject]
|
||||
private NotificationService NotificationService { get; set; } = default!;
|
||||
|
||||
private List<ManualSyncRequestViewModel> _requests = new();
|
||||
private List<PipelineInfoViewModel> _pipelines = new();
|
||||
private RadzenDataGrid<ManualSyncRequestViewModel>? _grid;
|
||||
private bool _isLoading = true;
|
||||
private bool _isReloadingPipelines;
|
||||
private bool _showPendingOnly = true;
|
||||
private string? _errorMessage;
|
||||
|
||||
/// <summary>
|
||||
/// Loads initial data when the component is initialized.
|
||||
/// </summary>
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
await LoadPipelinesAsync();
|
||||
await LoadRequestsAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads the available pipelines for creating new requests.
|
||||
/// </summary>
|
||||
private async Task LoadPipelinesAsync()
|
||||
{
|
||||
var result = await ManualSyncApi.GetPipelinesAsync();
|
||||
result.Switch(
|
||||
pipelines => { _pipelines = pipelines.ToList(); },
|
||||
_ => { /* Not found - use empty list */ },
|
||||
_ => { /* Validation error - use empty list */ },
|
||||
_ => { /* Unauthorized - use empty list */ },
|
||||
_ => { /* Forbidden - use empty list */ },
|
||||
_ => { /* Error - use empty list */ }
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads the manual sync requests from the API.
|
||||
/// </summary>
|
||||
private async Task LoadRequestsAsync()
|
||||
{
|
||||
_isLoading = true;
|
||||
_errorMessage = null;
|
||||
StateHasChanged();
|
||||
|
||||
try
|
||||
{
|
||||
var result = await ManualSyncApi.GetRequestsAsync(_showPendingOnly);
|
||||
result.Switch(
|
||||
requests => { _requests = requests.ToList(); },
|
||||
_ => { _errorMessage = "Requests not found."; _requests = new(); },
|
||||
validation => { _errorMessage = string.Join("; ", validation.FieldErrors.SelectMany(e => e.Value)); },
|
||||
_ => { _errorMessage = "Session expired. Please login again."; },
|
||||
_ => { _errorMessage = "Access denied."; },
|
||||
error => { _errorMessage = error.Message; }
|
||||
);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_isLoading = false;
|
||||
StateHasChanged();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Opens the new sync request dialog.
|
||||
/// </summary>
|
||||
private async Task OpenNewRequestDialogAsync()
|
||||
{
|
||||
var result = await DialogService.OpenAsync<NewSyncRequestDialog>(
|
||||
"New Sync Request",
|
||||
new Dictionary<string, object> { { "Pipelines", _pipelines } },
|
||||
new DialogOptions
|
||||
{
|
||||
Width = "400px",
|
||||
Height = "auto",
|
||||
CloseDialogOnOverlayClick = false
|
||||
});
|
||||
|
||||
if (result is ManualSyncRequestViewModel createdRequest)
|
||||
{
|
||||
NotificationService.Notify(NotificationSeverity.Success, "Success", $"Sync request for {createdRequest.PipelineName} created.");
|
||||
await LoadRequestsAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancels a pending sync request after user confirmation.
|
||||
/// </summary>
|
||||
/// <param name="request">The request to cancel.</param>
|
||||
private async Task CancelRequestAsync(ManualSyncRequestViewModel request)
|
||||
{
|
||||
var confirmed = await DialogService.Confirm(
|
||||
$"Are you sure you want to cancel the {request.SyncType} sync request for {request.PipelineName}?",
|
||||
"Confirm Cancel",
|
||||
new ConfirmOptions
|
||||
{
|
||||
OkButtonText = "Cancel Request",
|
||||
CancelButtonText = "Keep Request"
|
||||
});
|
||||
|
||||
if (confirmed != true)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var result = await ManualSyncApi.CancelRequestAsync(request.Id, request.RowVersionBase64);
|
||||
var shouldRefresh = false;
|
||||
result.Switch(
|
||||
unit =>
|
||||
{
|
||||
NotificationService.Notify(NotificationSeverity.Success, "Success", "Sync request cancelled.");
|
||||
shouldRefresh = true;
|
||||
},
|
||||
notFound => { NotificationService.Notify(NotificationSeverity.Warning, "Warning", "Request not found."); },
|
||||
validation => { NotificationService.Notify(NotificationSeverity.Error, "Error", string.Join("; ", validation.FieldErrors.SelectMany(e => e.Value))); },
|
||||
unauthorized => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Session expired. Please login again."); },
|
||||
forbidden => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Access denied."); },
|
||||
error => { NotificationService.Notify(NotificationSeverity.Error, "Error", error.Message); }
|
||||
);
|
||||
|
||||
if (shouldRefresh)
|
||||
{
|
||||
await LoadRequestsAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles filter checkbox change.
|
||||
/// </summary>
|
||||
/// <param name="value">The new checkbox value.</param>
|
||||
private async Task OnFilterChanged(bool value)
|
||||
{
|
||||
_showPendingOnly = value;
|
||||
await LoadRequestsAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the badge style based on the request status.
|
||||
/// </summary>
|
||||
/// <param name="status">The request status.</param>
|
||||
/// <returns>The appropriate badge style.</returns>
|
||||
private static BadgeStyle GetBadgeStyle(string status) => status switch
|
||||
{
|
||||
"Pending" => BadgeStyle.Warning,
|
||||
"Completed" => BadgeStyle.Success,
|
||||
"Cancelled" => BadgeStyle.Light,
|
||||
_ => BadgeStyle.Light
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Reloads pipeline definitions from disk (Admin only).
|
||||
/// </summary>
|
||||
private async Task ReloadPipelinesAsync()
|
||||
{
|
||||
_isReloadingPipelines = true;
|
||||
StateHasChanged();
|
||||
|
||||
try
|
||||
{
|
||||
var result = await PipelineApi.ReloadPipelinesAsync();
|
||||
result.Switch(
|
||||
reloadResult =>
|
||||
{
|
||||
if (reloadResult.Success)
|
||||
{
|
||||
NotificationService.Notify(
|
||||
NotificationSeverity.Success,
|
||||
"Pipelines Reloaded",
|
||||
$"Loaded {reloadResult.PipelinesLoaded} pipelines (version {reloadResult.NewVersion})");
|
||||
|
||||
// Refresh the pipeline list for the dialog
|
||||
_ = LoadPipelinesAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
var errorMessages = string.Join("; ", reloadResult.Errors.SelectMany(e => e.Messages));
|
||||
NotificationService.Notify(
|
||||
NotificationSeverity.Error,
|
||||
"Reload Failed",
|
||||
$"Pipeline reload failed: {errorMessages}");
|
||||
}
|
||||
},
|
||||
_ => { NotificationService.Notify(NotificationSeverity.Warning, "Warning", "Pipeline endpoint not found."); },
|
||||
validation => { NotificationService.Notify(NotificationSeverity.Error, "Error", string.Join("; ", validation.FieldErrors.SelectMany(e => e.Value))); },
|
||||
_ => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Session expired. Please login again."); },
|
||||
_ => { NotificationService.Notify(NotificationSeverity.Error, "Access Denied", "You must be an Admin to reload pipelines."); },
|
||||
error => { NotificationService.Notify(NotificationSeverity.Error, "Error", error.Message); }
|
||||
);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_isReloadingPipelines = false;
|
||||
StateHasChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -55,6 +55,8 @@ builder.Services.AddScoped<ISearchApiClient, SearchApiClient>();
|
||||
builder.Services.AddScoped<ILookupApiClient, LookupApiClient>();
|
||||
builder.Services.AddScoped<IAuthApiClient, AuthApiClient>();
|
||||
builder.Services.AddScoped<IFileApiClient, FileApiClient>();
|
||||
builder.Services.AddScoped<IManualSyncApiClient, ManualSyncApiClient>();
|
||||
builder.Services.AddScoped<IPipelineApiClient, PipelineApiClient>();
|
||||
|
||||
// Search services
|
||||
builder.Services.AddScoped<ISearchValidationService, SearchValidationService>();
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
using System.Net.Http.Json;
|
||||
using JdeScoping.Core.ApiContracts;
|
||||
using JdeScoping.Core.ApiContracts.Results;
|
||||
using JdeScoping.Core.ViewModels;
|
||||
|
||||
namespace JdeScoping.Client.Services;
|
||||
|
||||
/// <summary>
|
||||
/// HTTP client implementation of IManualSyncApiClient.
|
||||
/// </summary>
|
||||
public class ManualSyncApiClient : ApiClientBase, IManualSyncApiClient
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ManualSyncApiClient"/> class.
|
||||
/// </summary>
|
||||
/// <param name="httpClient">The HTTP client for API requests.</param>
|
||||
public ManualSyncApiClient(HttpClient httpClient) : base(httpClient) { }
|
||||
|
||||
/// <summary>
|
||||
/// Gets manual sync requests, optionally filtered to pending only.
|
||||
/// </summary>
|
||||
/// <param name="pendingOnly">If true, returns only pending requests.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A list of manual sync request view models.</returns>
|
||||
public Task<ApiResult<IReadOnlyList<ManualSyncRequestViewModel>>> GetRequestsAsync(bool pendingOnly = false, CancellationToken ct = default)
|
||||
=> GetAsync<IReadOnlyList<ManualSyncRequestViewModel>>(ApiRoutes.ManualSync.GetRequests(pendingOnly), ct);
|
||||
|
||||
/// <summary>
|
||||
/// Gets all available pipelines for manual sync.
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A list of pipeline information view models.</returns>
|
||||
public Task<ApiResult<IReadOnlyList<PipelineInfoViewModel>>> GetPipelinesAsync(CancellationToken ct = default)
|
||||
=> GetAsync<IReadOnlyList<PipelineInfoViewModel>>(ApiRoutes.ManualSync.Pipelines, ct);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new manual sync request.
|
||||
/// </summary>
|
||||
/// <param name="request">The request details.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The created manual sync request view model.</returns>
|
||||
public Task<ApiResult<ManualSyncRequestViewModel>> CreateRequestAsync(CreateManualSyncRequestDto request, CancellationToken ct = default)
|
||||
=> PostAsync<ManualSyncRequestViewModel, CreateManualSyncRequestDto>(ApiRoutes.ManualSync.Base, request, ct);
|
||||
|
||||
/// <summary>
|
||||
/// Cancels a pending manual sync request.
|
||||
/// </summary>
|
||||
/// <param name="id">The request ID.</param>
|
||||
/// <param name="rowVersionBase64">The row version for optimistic concurrency.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A unit result indicating success or failure.</returns>
|
||||
public Task<ApiResult<Unit>> CancelRequestAsync(int id, string rowVersionBase64, CancellationToken ct = default)
|
||||
{
|
||||
var cancelDto = new CancelManualSyncRequestDto { RowVersionBase64 = rowVersionBase64 };
|
||||
return PostAsync<Unit, CancelManualSyncRequestDto>(ApiRoutes.ManualSync.GetCancel(id), cancelDto, ct);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// DTO for cancelling a manual sync request.
|
||||
/// </summary>
|
||||
internal class CancelManualSyncRequestDto
|
||||
{
|
||||
/// <summary>
|
||||
/// The row version for optimistic concurrency (Base64 encoded).
|
||||
/// </summary>
|
||||
public string RowVersionBase64 { get; set; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
using JdeScoping.Core.ApiContracts;
|
||||
using JdeScoping.Core.ApiContracts.Results;
|
||||
using JdeScoping.Core.ViewModels;
|
||||
|
||||
namespace JdeScoping.Client.Services;
|
||||
|
||||
/// <summary>
|
||||
/// HTTP client implementation of IPipelineApiClient.
|
||||
/// </summary>
|
||||
public class PipelineApiClient : ApiClientBase, IPipelineApiClient
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PipelineApiClient"/> class.
|
||||
/// </summary>
|
||||
/// <param name="httpClient">The HTTP client for API requests.</param>
|
||||
public PipelineApiClient(HttpClient httpClient) : base(httpClient) { }
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<ApiResult<IReadOnlyList<PipelineInfoViewModel>>> GetPipelinesAsync(CancellationToken ct = default)
|
||||
=> GetAsync<IReadOnlyList<PipelineInfoViewModel>>(ApiRoutes.Pipeline.Base, ct);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<ApiResult<PipelineRegistryStatusViewModel>> GetStatusAsync(CancellationToken ct = default)
|
||||
=> GetAsync<PipelineRegistryStatusViewModel>(ApiRoutes.Pipeline.Status, ct);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<ApiResult<PipelineReloadResultViewModel>> ReloadPipelinesAsync(CancellationToken ct = default)
|
||||
=> PostAsync<PipelineReloadResultViewModel>(ApiRoutes.Pipeline.Reload, ct);
|
||||
}
|
||||
@@ -152,4 +152,44 @@ public static class ApiRoutes
|
||||
$"api/refresh-status?minDT={minDt:yyyy-MM-dd}&maxDT={maxDt:yyyy-MM-dd}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Routes for manual sync API endpoints.
|
||||
/// </summary>
|
||||
public static class ManualSync
|
||||
{
|
||||
/// <summary>Base route for manual sync endpoints.</summary>
|
||||
public const string Base = "api/manual-sync";
|
||||
|
||||
/// <summary>Route for pipeline list.</summary>
|
||||
public const string Pipelines = "api/manual-sync/pipelines";
|
||||
|
||||
/// <summary>Route template for cancelling a request (use in controller attributes).</summary>
|
||||
public const string Cancel = "{id:int}/cancel";
|
||||
|
||||
/// <summary>Builds the route to get requests with optional filter.</summary>
|
||||
/// <param name="pendingOnly">If true, returns only pending requests.</param>
|
||||
/// <returns>The formatted route.</returns>
|
||||
public static string GetRequests(bool pendingOnly = false) =>
|
||||
pendingOnly ? $"{Base}?pendingOnly=true" : Base;
|
||||
|
||||
/// <summary>Builds the route to cancel a specific request.</summary>
|
||||
/// <param name="id">The request ID.</param>
|
||||
/// <returns>The formatted route.</returns>
|
||||
public static string GetCancel(int id) => $"{Base}/{id}/cancel";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Routes for pipeline API endpoints.
|
||||
/// </summary>
|
||||
public static class Pipeline
|
||||
{
|
||||
/// <summary>Base route for pipeline endpoints.</summary>
|
||||
public const string Base = "api/pipelines";
|
||||
|
||||
/// <summary>Route for pipeline status.</summary>
|
||||
public const string Status = "api/pipelines/status";
|
||||
|
||||
/// <summary>Route to reload pipelines.</summary>
|
||||
public const string Reload = "api/pipelines/reload";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
using JdeScoping.Core.ApiContracts.Results;
|
||||
using JdeScoping.Core.ViewModels;
|
||||
|
||||
namespace JdeScoping.Core.ApiContracts;
|
||||
|
||||
/// <summary>
|
||||
/// Client contract for manual sync API operations.
|
||||
/// </summary>
|
||||
public interface IManualSyncApiClient
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets manual sync requests, optionally filtered to pending only.
|
||||
/// </summary>
|
||||
/// <param name="pendingOnly">If true, returns only pending requests.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A list of manual sync request view models.</returns>
|
||||
Task<ApiResult<IReadOnlyList<ManualSyncRequestViewModel>>> GetRequestsAsync(bool pendingOnly = false, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets all available pipelines for manual sync.
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A list of pipeline information view models.</returns>
|
||||
Task<ApiResult<IReadOnlyList<PipelineInfoViewModel>>> GetPipelinesAsync(CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new manual sync request.
|
||||
/// </summary>
|
||||
/// <param name="request">The request details.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The created manual sync request view model.</returns>
|
||||
Task<ApiResult<ManualSyncRequestViewModel>> CreateRequestAsync(CreateManualSyncRequestDto request, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Cancels a pending manual sync request.
|
||||
/// </summary>
|
||||
/// <param name="id">The request ID.</param>
|
||||
/// <param name="rowVersionBase64">The row version for optimistic concurrency.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A unit result indicating success or failure.</returns>
|
||||
Task<ApiResult<Unit>> CancelRequestAsync(int id, string rowVersionBase64, CancellationToken ct = default);
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
using JdeScoping.Core.ApiContracts.Results;
|
||||
using JdeScoping.Core.ViewModels;
|
||||
|
||||
namespace JdeScoping.Core.ApiContracts;
|
||||
|
||||
/// <summary>
|
||||
/// Client contract for pipeline API operations.
|
||||
/// </summary>
|
||||
public interface IPipelineApiClient
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets all enabled pipelines.
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A list of pipeline information view models.</returns>
|
||||
Task<ApiResult<IReadOnlyList<PipelineInfoViewModel>>> GetPipelinesAsync(CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the pipeline registry status.
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The registry status.</returns>
|
||||
Task<ApiResult<PipelineRegistryStatusViewModel>> GetStatusAsync(CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Reloads all pipeline definitions from disk.
|
||||
/// Requires Admin role.
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The reload result.</returns>
|
||||
Task<ApiResult<PipelineReloadResultViewModel>> ReloadPipelinesAsync(CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// View model for pipeline registry status.
|
||||
/// </summary>
|
||||
public class PipelineRegistryStatusViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the current registry version.
|
||||
/// </summary>
|
||||
public int Version { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the timestamp of the last successful load.
|
||||
/// </summary>
|
||||
public DateTime? LastLoadedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the total number of pipelines.
|
||||
/// </summary>
|
||||
public int TotalPipelines { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of enabled pipelines.
|
||||
/// </summary>
|
||||
public int EnabledPipelines { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// View model for pipeline reload result.
|
||||
/// </summary>
|
||||
public class PipelineReloadResultViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the reload was successful.
|
||||
/// </summary>
|
||||
public bool Success { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of pipelines loaded.
|
||||
/// </summary>
|
||||
public int PipelinesLoaded { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of pipelines skipped.
|
||||
/// </summary>
|
||||
public int PipelinesSkipped { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the previous version.
|
||||
/// </summary>
|
||||
public int PreviousVersion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the new version.
|
||||
/// </summary>
|
||||
public int NewVersion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the list of errors.
|
||||
/// </summary>
|
||||
public List<PipelineLoadErrorViewModel> Errors { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// View model for pipeline load error.
|
||||
/// </summary>
|
||||
public class PipelineLoadErrorViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the file name.
|
||||
/// </summary>
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the pipeline name.
|
||||
/// </summary>
|
||||
public string PipelineName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the error type.
|
||||
/// </summary>
|
||||
public string ErrorType { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the error messages.
|
||||
/// </summary>
|
||||
public List<string> Messages { get; set; } = [];
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace JdeScoping.Core.ViewModels;
|
||||
|
||||
/// <summary>
|
||||
/// Request to create a new manual sync request.
|
||||
/// </summary>
|
||||
public class CreateManualSyncRequestDto
|
||||
{
|
||||
/// <summary>
|
||||
/// The name of the pipeline to sync.
|
||||
/// </summary>
|
||||
public string PipelineName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The type of sync to perform (mass, daily, hourly).
|
||||
/// </summary>
|
||||
public string SyncType { get; set; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
namespace JdeScoping.Core.ViewModels;
|
||||
|
||||
/// <summary>
|
||||
/// View model for a manual sync request.
|
||||
/// </summary>
|
||||
public class ManualSyncRequestViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// The unique identifier for the sync request.
|
||||
/// </summary>
|
||||
public int Id { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The name of the pipeline being synced.
|
||||
/// </summary>
|
||||
public string PipelineName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The type of sync (mass, daily, hourly).
|
||||
/// </summary>
|
||||
public string SyncType { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The date and time the request was made.
|
||||
/// </summary>
|
||||
public DateTime RequestDT { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The username of the person who requested the sync.
|
||||
/// </summary>
|
||||
public string RequestedBy { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The date and time the sync completed, if applicable.
|
||||
/// </summary>
|
||||
public DateTime? CompletedDT { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The date and time the sync was cancelled, if applicable.
|
||||
/// </summary>
|
||||
public DateTime? CancelDT { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The username of the person who cancelled the sync, if applicable.
|
||||
/// </summary>
|
||||
public string? CancelledBy { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The current status of the sync request.
|
||||
/// </summary>
|
||||
public string Status { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The row version for optimistic concurrency (Base64 encoded).
|
||||
/// </summary>
|
||||
public string RowVersionBase64 { get; set; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace JdeScoping.Core.ViewModels;
|
||||
|
||||
/// <summary>
|
||||
/// View model for pipeline information.
|
||||
/// </summary>
|
||||
public class PipelineInfoViewModel
|
||||
{
|
||||
/// <summary>
|
||||
/// The name of the pipeline.
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The sync types supported by this pipeline.
|
||||
/// </summary>
|
||||
public List<string> SupportedSyncTypes { get; set; } = new();
|
||||
}
|
||||
@@ -52,6 +52,9 @@ public static class DataAccessDependencyInjection
|
||||
services.AddScoped<IWorkOrderTraversalService, WorkOrderTraversalService>();
|
||||
services.AddScoped<ISearchProcessor, SearchProcessor>();
|
||||
|
||||
// Register manual sync request service (scoped - per request lifetime)
|
||||
services.AddScoped<IManualSyncRequestService, ManualSyncRequestService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\JdeScoping.Core\JdeScoping.Core.csproj" />
|
||||
<ProjectReference Include="..\JdeScoping.Domain\JdeScoping.Domain.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
using JdeScoping.Domain.Models;
|
||||
|
||||
namespace JdeScoping.DataAccess.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Service for managing manual data sync requests.
|
||||
/// </summary>
|
||||
public interface IManualSyncRequestService
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets all manual sync requests, optionally filtered to pending only.
|
||||
/// </summary>
|
||||
/// <param name="pendingOnly">If true, returns only pending requests.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A read-only list of manual sync requests.</returns>
|
||||
Task<IReadOnlyList<ManualSyncRequest>> GetRequestsAsync(
|
||||
bool pendingOnly = false,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the next pending request in FIFO order (for processor).
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The next pending request, or null if none exists.</returns>
|
||||
Task<ManualSyncRequest?> GetNextPendingRequestAsync(CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new manual sync request.
|
||||
/// </summary>
|
||||
/// <param name="pipelineName">The name of the ETL pipeline to sync.</param>
|
||||
/// <param name="syncType">The type of sync (mass, daily, hourly).</param>
|
||||
/// <param name="requestedBy">The username of the requester.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The created manual sync request.</returns>
|
||||
Task<ManualSyncRequest> CreateRequestAsync(
|
||||
string pipelineName,
|
||||
string syncType,
|
||||
string requestedBy,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Cancels a pending request using optimistic concurrency.
|
||||
/// Returns true if cancelled, false if already completed/cancelled.
|
||||
/// </summary>
|
||||
/// <param name="id">The request ID.</param>
|
||||
/// <param name="cancelledBy">The username of the user cancelling the request.</param>
|
||||
/// <param name="rowVersion">The row version for optimistic concurrency.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>True if the request was cancelled; false if already completed or cancelled.</returns>
|
||||
Task<bool> CancelRequestAsync(
|
||||
int id,
|
||||
string cancelledBy,
|
||||
byte[] rowVersion,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Marks a request as completed using optimistic concurrency.
|
||||
/// Called by the sync processor.
|
||||
/// </summary>
|
||||
/// <param name="id">The request ID.</param>
|
||||
/// <param name="rowVersion">The row version for optimistic concurrency.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>True if the request was marked completed; false if already completed or cancelled.</returns>
|
||||
Task<bool> CompleteRequestAsync(
|
||||
int id,
|
||||
byte[] rowVersion,
|
||||
CancellationToken ct = default);
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
using Dapper;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.Domain.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace JdeScoping.DataAccess.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Service implementation for managing manual data sync requests.
|
||||
/// </summary>
|
||||
public sealed class ManualSyncRequestService : IManualSyncRequestService
|
||||
{
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly ILogger<ManualSyncRequestService> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ManualSyncRequestService"/> class.
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">The database connection factory.</param>
|
||||
/// <param name="logger">The logger instance.</param>
|
||||
public ManualSyncRequestService(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
ILogger<ManualSyncRequestService> logger)
|
||||
{
|
||||
_connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<IReadOnlyList<ManualSyncRequest>> GetRequestsAsync(
|
||||
bool pendingOnly = false,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
_logger.LogDebug("Getting manual sync requests (pendingOnly: {PendingOnly})", pendingOnly);
|
||||
|
||||
const string sqlAll = """
|
||||
SELECT ID, PipelineName, SyncType, RequestDT, RequestedBy,
|
||||
CompletedDT, CancelDT, CancelledBy, RowVersion
|
||||
FROM dbo.ManualSyncRequest
|
||||
ORDER BY RequestDT DESC
|
||||
""";
|
||||
|
||||
const string sqlPending = """
|
||||
SELECT ID, PipelineName, SyncType, RequestDT, RequestedBy,
|
||||
CompletedDT, CancelDT, CancelledBy, RowVersion
|
||||
FROM dbo.ManualSyncRequest
|
||||
WHERE CompletedDT IS NULL AND CancelDT IS NULL
|
||||
ORDER BY RequestDT ASC
|
||||
""";
|
||||
|
||||
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
|
||||
var results = await connection.QueryAsync<ManualSyncRequest>(
|
||||
pendingOnly ? sqlPending : sqlAll);
|
||||
|
||||
var list = results.ToList();
|
||||
_logger.LogDebug("Retrieved {Count} manual sync requests", list.Count);
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<ManualSyncRequest?> GetNextPendingRequestAsync(CancellationToken ct = default)
|
||||
{
|
||||
_logger.LogDebug("Getting next pending manual sync request");
|
||||
|
||||
const string sql = """
|
||||
SELECT TOP 1 ID, PipelineName, SyncType, RequestDT, RequestedBy,
|
||||
CompletedDT, CancelDT, CancelledBy, RowVersion
|
||||
FROM dbo.ManualSyncRequest
|
||||
WHERE CompletedDT IS NULL AND CancelDT IS NULL
|
||||
ORDER BY RequestDT ASC
|
||||
""";
|
||||
|
||||
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
|
||||
var result = await connection.QueryFirstOrDefaultAsync<ManualSyncRequest>(sql);
|
||||
|
||||
if (result != null)
|
||||
{
|
||||
_logger.LogDebug("Found pending request ID {Id} for pipeline {Pipeline}",
|
||||
result.Id, result.PipelineName);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("No pending manual sync requests found");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<ManualSyncRequest> CreateRequestAsync(
|
||||
string pipelineName,
|
||||
string syncType,
|
||||
string requestedBy,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Creating manual sync request for pipeline {Pipeline}, type {SyncType}, by {User}",
|
||||
pipelineName, syncType, requestedBy);
|
||||
|
||||
const string sql = """
|
||||
INSERT INTO dbo.ManualSyncRequest (PipelineName, SyncType, RequestedBy)
|
||||
OUTPUT INSERTED.ID, INSERTED.PipelineName, INSERTED.SyncType,
|
||||
INSERTED.RequestDT, INSERTED.RequestedBy,
|
||||
INSERTED.CompletedDT, INSERTED.CancelDT, INSERTED.CancelledBy,
|
||||
INSERTED.RowVersion
|
||||
VALUES (@PipelineName, @SyncType, @RequestedBy)
|
||||
""";
|
||||
|
||||
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
|
||||
var result = await connection.QuerySingleAsync<ManualSyncRequest>(sql, new
|
||||
{
|
||||
PipelineName = pipelineName,
|
||||
SyncType = syncType,
|
||||
RequestedBy = requestedBy
|
||||
});
|
||||
|
||||
_logger.LogInformation("Created manual sync request with ID {Id}", result.Id);
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<bool> CancelRequestAsync(
|
||||
int id,
|
||||
string cancelledBy,
|
||||
byte[] rowVersion,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Cancelling manual sync request ID {Id} by {User}",
|
||||
id, cancelledBy);
|
||||
|
||||
const string sql = """
|
||||
UPDATE dbo.ManualSyncRequest
|
||||
SET CancelDT = @CancelDT, CancelledBy = @CancelledBy
|
||||
WHERE ID = @Id AND RowVersion = @RowVersion
|
||||
AND CompletedDT IS NULL AND CancelDT IS NULL
|
||||
""";
|
||||
|
||||
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
|
||||
var affectedRows = await connection.ExecuteAsync(sql, new
|
||||
{
|
||||
Id = id,
|
||||
CancelDT = DateTime.UtcNow,
|
||||
CancelledBy = cancelledBy,
|
||||
RowVersion = rowVersion
|
||||
});
|
||||
|
||||
var success = affectedRows > 0;
|
||||
|
||||
if (success)
|
||||
{
|
||||
_logger.LogInformation("Successfully cancelled manual sync request ID {Id}", id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Failed to cancel manual sync request ID {Id} - already completed/cancelled or version mismatch",
|
||||
id);
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<bool> CompleteRequestAsync(
|
||||
int id,
|
||||
byte[] rowVersion,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
_logger.LogInformation("Completing manual sync request ID {Id}", id);
|
||||
|
||||
const string sql = """
|
||||
UPDATE dbo.ManualSyncRequest
|
||||
SET CompletedDT = @CompletedDT
|
||||
WHERE ID = @Id AND RowVersion = @RowVersion
|
||||
AND CompletedDT IS NULL AND CancelDT IS NULL
|
||||
""";
|
||||
|
||||
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
|
||||
var affectedRows = await connection.ExecuteAsync(sql, new
|
||||
{
|
||||
Id = id,
|
||||
CompletedDT = DateTime.UtcNow,
|
||||
RowVersion = rowVersion
|
||||
});
|
||||
|
||||
var success = affectedRows > 0;
|
||||
|
||||
if (success)
|
||||
{
|
||||
_logger.LogInformation("Successfully completed manual sync request ID {Id}", id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Failed to complete manual sync request ID {Id} - already completed/cancelled or version mismatch",
|
||||
id);
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace JdeScoping.DataSync.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for the pipeline destination.
|
||||
/// </summary>
|
||||
public class DestinationElement
|
||||
{
|
||||
/// <summary>
|
||||
/// Target table name in the cache database.
|
||||
/// </summary>
|
||||
public string Table { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Columns used to match existing records for upsert.
|
||||
/// </summary>
|
||||
public List<string> MatchColumns { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Columns to exclude from UPDATE operations.
|
||||
/// </summary>
|
||||
public List<string> ExcludeFromUpdate { get; set; } = [];
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
namespace JdeScoping.DataSync.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Represents an ETL pipeline definition loaded from a JSON file.
|
||||
/// </summary>
|
||||
public class EtlPipelineConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique name of the pipeline (must match filename, case-insensitive).
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether the pipeline is enabled for execution.
|
||||
/// </summary>
|
||||
public bool IsEnabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// If true, pipeline can only be triggered via ManualSyncRequest.
|
||||
/// No interval validation is required for manual-only pipelines.
|
||||
/// </summary>
|
||||
public bool IsManualOnly { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Interval for mass sync in minutes. Null if mass sync not supported.
|
||||
/// </summary>
|
||||
public int? MassSyncIntervalMinutes { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Interval for daily sync in minutes. Null if daily sync not supported.
|
||||
/// </summary>
|
||||
public int? DailySyncIntervalMinutes { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Interval for hourly sync in minutes. Null if hourly sync not supported.
|
||||
/// </summary>
|
||||
public int? HourlySyncIntervalMinutes { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Scripts to run before the main sync. Optional.
|
||||
/// </summary>
|
||||
public List<ScriptElement> PreScripts { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// The data source configuration. Required.
|
||||
/// </summary>
|
||||
public SourceElement Source { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Data transformations to apply. Optional.
|
||||
/// </summary>
|
||||
public List<TransformElement> Transforms { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// The destination configuration. Required.
|
||||
/// </summary>
|
||||
public DestinationElement Destination { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Scripts to run after the main sync. Optional.
|
||||
/// </summary>
|
||||
public List<ScriptElement> PostScripts { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the pipeline supports mass sync.
|
||||
/// </summary>
|
||||
public bool SupportsMassSync => MassSyncIntervalMinutes.HasValue;
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the pipeline supports daily sync.
|
||||
/// </summary>
|
||||
public bool SupportsDailySync => DailySyncIntervalMinutes.HasValue;
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the pipeline supports hourly sync.
|
||||
/// </summary>
|
||||
public bool SupportsHourlySync => HourlySyncIntervalMinutes.HasValue;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
namespace JdeScoping.DataSync.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a query parameter.
|
||||
/// </summary>
|
||||
public class ParameterElement
|
||||
{
|
||||
/// <summary>
|
||||
/// Parameter name as used in query (e.g., ":dateUpdated").
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Format conversion (jdeJulian, jdeTime, etc.).
|
||||
/// </summary>
|
||||
public string? Format { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Source of the value (offset = from last sync time).
|
||||
/// </summary>
|
||||
public string Source { get; set; } = "offset";
|
||||
|
||||
/// <summary>
|
||||
/// Static value if source is not offset.
|
||||
/// </summary>
|
||||
public string? Value { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace JdeScoping.DataSync.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a pre/post script.
|
||||
/// </summary>
|
||||
public class ScriptElement
|
||||
{
|
||||
/// <summary>
|
||||
/// Connection identifier for script execution.
|
||||
/// </summary>
|
||||
public string Connection { get; set; } = "lotfinder";
|
||||
|
||||
/// <summary>
|
||||
/// SQL script to execute.
|
||||
/// </summary>
|
||||
public string Script { get; set; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
namespace JdeScoping.DataSync.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for the pipeline data source.
|
||||
/// </summary>
|
||||
public class SourceElement
|
||||
{
|
||||
/// <summary>
|
||||
/// Connection identifier (jde, cms, giw, lotfinder).
|
||||
/// </summary>
|
||||
public string Connection { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Query for incremental syncs (daily/hourly).
|
||||
/// </summary>
|
||||
public string Query { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Query for mass sync. Falls back to Query if not specified.
|
||||
/// </summary>
|
||||
public string? MassQuery { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Query parameters with format and source configuration.
|
||||
/// </summary>
|
||||
public Dictionary<string, ParameterElement> Parameters { get; set; } = new();
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace JdeScoping.DataSync.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a data transformation.
|
||||
/// </summary>
|
||||
public class TransformElement
|
||||
{
|
||||
/// <summary>
|
||||
/// Type of transformation (ColumnDrop, ColumnRename, JdeDate, Regex, etc.).
|
||||
/// </summary>
|
||||
public string TransformType { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Transform-specific configuration as raw JSON.
|
||||
/// Using JsonElement avoids Dictionary<string, object> deserialization issues
|
||||
/// where values would become JsonElement anyway without custom converters.
|
||||
/// </summary>
|
||||
public JsonElement? Config { get; set; }
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
using JdeScoping.DataSync.Models;
|
||||
|
||||
namespace JdeScoping.DataSync.Contracts;
|
||||
|
||||
/// <summary>
|
||||
@@ -11,4 +13,12 @@ public interface ISyncOrchestrator
|
||||
/// <param name="cancellationToken">Cancellation token for graceful shutdown.</param>
|
||||
/// <returns>A task representing the async operation.</returns>
|
||||
Task ExecutePendingSyncsAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Executes a single sync operation (used for manual sync requests).
|
||||
/// </summary>
|
||||
/// <param name="task">The sync task to execute.</param>
|
||||
/// <param name="cancellationToken">Cancellation token for graceful shutdown.</param>
|
||||
/// <returns>A task representing the async operation.</returns>
|
||||
Task ExecuteSingleSyncAsync(DataUpdateTask task, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -42,6 +42,13 @@ public static class DataSyncDependencyInjection
|
||||
// Pipeline factory (new ETL infrastructure)
|
||||
services.AddSingleton<IEtlPipelineFactory, EtlPipelineFactory>();
|
||||
|
||||
// Pipeline registry services (new hot-reload infrastructure)
|
||||
services.AddSingleton<IPipelineValidator, PipelineValidator>();
|
||||
services.AddSingleton<IPipelineRegistry, PipelineRegistry>();
|
||||
|
||||
// Pipeline registry initializer - runs before WorkProcessor to ensure pipelines load first
|
||||
services.AddHostedService<PipelineRegistryInitializer>();
|
||||
|
||||
// Register hosted service (WorkProcessor combines data sync and search processing)
|
||||
services.AddHostedService<WorkProcessor>();
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Options;
|
||||
|
||||
namespace JdeScoping.DataSync.Models;
|
||||
@@ -19,7 +20,7 @@ public class DataUpdateTask
|
||||
public required string TableName { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Source system: "JDE" or "CMS".
|
||||
/// Source system: "JDE", "CMS", "GIW", or "LOTFINDER".
|
||||
/// </summary>
|
||||
public required string SourceSystem { get; init; }
|
||||
|
||||
@@ -40,9 +41,15 @@ public class DataUpdateTask
|
||||
public DateTime? MinimumDt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The data source configuration for this task.
|
||||
/// The pipeline configuration for this task (new format).
|
||||
/// </summary>
|
||||
public required DataSourceConfig Config { get; init; }
|
||||
public EtlPipelineConfig? Pipeline { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The data source configuration for this task (legacy format - will be removed).
|
||||
/// </summary>
|
||||
[Obsolete("Use Pipeline instead. This property exists for backward compatibility during migration.")]
|
||||
public DataSourceConfig? Config { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a unique key for logging purposes.
|
||||
@@ -50,13 +57,47 @@ public class DataUpdateTask
|
||||
public string LogKey => $"{TableName}_{UpdateType}_{OperationId:N}";
|
||||
|
||||
/// <summary>
|
||||
/// Gets the schedule configuration for this update type.
|
||||
/// Gets the interval in minutes for this update type from the pipeline config.
|
||||
/// </summary>
|
||||
public ScheduleConfig ScheduleConfig => UpdateType switch
|
||||
public int? IntervalMinutes => Pipeline != null ? UpdateType switch
|
||||
{
|
||||
UpdateTypes.Mass => Config.MassConfig,
|
||||
UpdateTypes.Daily => Config.DailyConfig,
|
||||
UpdateTypes.Hourly => Config.HourlyConfig,
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(UpdateType))
|
||||
};
|
||||
UpdateTypes.Mass => Pipeline.MassSyncIntervalMinutes,
|
||||
UpdateTypes.Daily => Pipeline.DailySyncIntervalMinutes,
|
||||
UpdateTypes.Hourly => Pipeline.HourlySyncIntervalMinutes,
|
||||
_ => null
|
||||
} : null;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a DataUpdateTask from an EtlPipelineConfig.
|
||||
/// </summary>
|
||||
public static DataUpdateTask FromPipeline(
|
||||
EtlPipelineConfig pipeline,
|
||||
UpdateTypes updateType,
|
||||
DateTime? minimumDt = null)
|
||||
{
|
||||
return new DataUpdateTask
|
||||
{
|
||||
TableName = pipeline.Destination.Table,
|
||||
SourceSystem = MapConnectionToSourceSystem(pipeline.Source.Connection),
|
||||
SourceData = pipeline.Name,
|
||||
UpdateType = updateType,
|
||||
MinimumDt = minimumDt,
|
||||
Pipeline = pipeline
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps connection identifier to source system name.
|
||||
/// </summary>
|
||||
private static string MapConnectionToSourceSystem(string connection)
|
||||
{
|
||||
return connection.ToUpperInvariant() switch
|
||||
{
|
||||
"JDE" => "JDE",
|
||||
"CMS" => "CMS",
|
||||
"GIW" => "GIW",
|
||||
"LOTFINDER" => "LOTFINDER",
|
||||
_ => connection.ToUpperInvariant()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +58,18 @@ public class DataSyncOptions
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Directory containing pipeline.*.json files.
|
||||
/// Resolved relative to content root.
|
||||
/// </summary>
|
||||
public string PipelinesDirectory { get; set; } = "Pipelines";
|
||||
|
||||
/// <summary>
|
||||
/// If true (default), startup fails if any enabled pipeline is invalid.
|
||||
/// If false, invalid enabled pipelines are skipped with warnings.
|
||||
/// </summary>
|
||||
public bool StrictPipelineValidation { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Per-table data source configurations.
|
||||
/// </summary>
|
||||
|
||||
@@ -34,4 +34,12 @@ public class WorkProcessorOptions
|
||||
/// </summary>
|
||||
[Range(1, 365)]
|
||||
public int PurgeRetentionDays { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum manual sync requests to process per work cycle.
|
||||
/// Prevents manual requests from starving scheduled syncs.
|
||||
/// Default: 5. Set to 0 for unlimited (not recommended).
|
||||
/// </summary>
|
||||
[Range(0, 100)]
|
||||
public int MaxManualRequestsPerCycle { get; set; } = 5;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Registry for ETL pipeline definitions with hot reload support.
|
||||
/// </summary>
|
||||
public interface IPipelineRegistry
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets all loaded pipelines (immutable snapshot).
|
||||
/// </summary>
|
||||
IReadOnlyList<EtlPipelineConfig> GetAllPipelines();
|
||||
|
||||
/// <summary>
|
||||
/// Gets all enabled pipelines (immutable snapshot).
|
||||
/// </summary>
|
||||
IReadOnlyList<EtlPipelineConfig> GetEnabledPipelines();
|
||||
|
||||
/// <summary>
|
||||
/// Gets a pipeline by name (case-insensitive).
|
||||
/// </summary>
|
||||
/// <param name="name">The pipeline name to find.</param>
|
||||
/// <returns>The pipeline if found, or null.</returns>
|
||||
EtlPipelineConfig? GetPipeline(string name);
|
||||
|
||||
/// <summary>
|
||||
/// Validates that a pipeline and sync type combination is valid.
|
||||
/// </summary>
|
||||
/// <param name="pipelineName">The pipeline name.</param>
|
||||
/// <param name="syncType">The sync type (mass, daily, hourly).</param>
|
||||
/// <returns>True if the combination is valid.</returns>
|
||||
bool IsValidPipelineAndSyncType(string pipelineName, string syncType);
|
||||
|
||||
/// <summary>
|
||||
/// Reloads all pipelines from disk.
|
||||
/// Returns validation results for each file.
|
||||
/// Reload is atomic: on any enabled pipeline failure, keeps previous snapshot.
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The reload result.</returns>
|
||||
Task<PipelineReloadResult> ReloadAsync(CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current registry version (increments on successful reload).
|
||||
/// </summary>
|
||||
int Version { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the timestamp of the last successful load.
|
||||
/// </summary>
|
||||
DateTime? LastLoadedAt { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a pipeline reload operation.
|
||||
/// </summary>
|
||||
public class PipelineReloadResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the reload was successful.
|
||||
/// </summary>
|
||||
public bool Success { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of pipelines loaded.
|
||||
/// </summary>
|
||||
public int PipelinesLoaded { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of pipelines skipped due to errors.
|
||||
/// </summary>
|
||||
public int PipelinesSkipped { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the previous version before reload.
|
||||
/// </summary>
|
||||
public int PreviousVersion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the new version after reload (unchanged if reload failed).
|
||||
/// </summary>
|
||||
public int NewVersion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the list of validation errors.
|
||||
/// </summary>
|
||||
public List<PipelineLoadError> Errors { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents an error that occurred while loading a pipeline.
|
||||
/// </summary>
|
||||
public class PipelineLoadError
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the file name that had the error.
|
||||
/// </summary>
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the pipeline name (if parseable).
|
||||
/// </summary>
|
||||
public string PipelineName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the error type (parse, validation, file).
|
||||
/// </summary>
|
||||
public string ErrorType { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the error messages.
|
||||
/// </summary>
|
||||
public List<string> Messages { get; set; } = [];
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Validates pipeline definitions.
|
||||
/// </summary>
|
||||
public interface IPipelineValidator
|
||||
{
|
||||
/// <summary>
|
||||
/// Validates a pipeline definition.
|
||||
/// </summary>
|
||||
/// <param name="pipeline">The pipeline to validate.</param>
|
||||
/// <param name="fileName">The source file name for error messages.</param>
|
||||
/// <returns>Validation result with errors and warnings.</returns>
|
||||
PipelineValidationResult Validate(EtlPipelineConfig pipeline, string fileName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of validating a single pipeline.
|
||||
/// </summary>
|
||||
public class PipelineValidationResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the pipeline is valid.
|
||||
/// </summary>
|
||||
public bool IsValid { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the list of validation errors.
|
||||
/// </summary>
|
||||
public List<string> Errors { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the list of validation warnings.
|
||||
/// </summary>
|
||||
public List<string> Warnings { get; set; } = [];
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
using System.Text.Json;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Registry for ETL pipeline definitions with hot reload support.
|
||||
/// Uses immutable snapshots for thread-safe reads.
|
||||
/// </summary>
|
||||
public class PipelineRegistry : IPipelineRegistry
|
||||
{
|
||||
private readonly IOptions<DataSyncOptions> _options;
|
||||
private readonly IPipelineValidator _validator;
|
||||
private readonly ILogger<PipelineRegistry> _logger;
|
||||
private readonly IHostEnvironment _environment;
|
||||
|
||||
/// <summary>
|
||||
/// Immutable snapshot - swapped atomically.
|
||||
/// </summary>
|
||||
private volatile PipelineSnapshot _snapshot = PipelineSnapshot.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Serializes reload operations (only one reload at a time).
|
||||
/// </summary>
|
||||
private readonly SemaphoreSlim _reloadLock = new(1, 1);
|
||||
|
||||
/// <summary>
|
||||
/// Version incremented via Interlocked.
|
||||
/// </summary>
|
||||
private int _version;
|
||||
|
||||
private static readonly JsonSerializerOptions PipelineJsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
AllowTrailingCommas = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = true,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PipelineRegistry"/> class.
|
||||
/// </summary>
|
||||
public PipelineRegistry(
|
||||
IOptions<DataSyncOptions> options,
|
||||
IPipelineValidator validator,
|
||||
ILogger<PipelineRegistry> logger,
|
||||
IHostEnvironment environment)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_validator = validator ?? throw new ArgumentNullException(nameof(validator));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IReadOnlyList<EtlPipelineConfig> GetAllPipelines() => _snapshot.AllPipelines;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IReadOnlyList<EtlPipelineConfig> GetEnabledPipelines() => _snapshot.EnabledPipelines;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public EtlPipelineConfig? GetPipeline(string name) => _snapshot.GetByName(name);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public int Version => _version;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public DateTime? LastLoadedAt => _snapshot.LoadedAt == default ? null : _snapshot.LoadedAt;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool IsValidPipelineAndSyncType(string pipelineName, string syncType)
|
||||
{
|
||||
var pipeline = GetPipeline(pipelineName);
|
||||
if (pipeline == null || !pipeline.IsEnabled)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return syncType.ToLowerInvariant() switch
|
||||
{
|
||||
"mass" => pipeline.SupportsMassSync,
|
||||
"daily" => pipeline.SupportsDailySync,
|
||||
"hourly" => pipeline.SupportsHourlySync,
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<PipelineReloadResult> ReloadAsync(CancellationToken ct = default)
|
||||
{
|
||||
await _reloadLock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var newSnapshot = await LoadSnapshotAsync(ct);
|
||||
var previousVersion = _version;
|
||||
|
||||
// Atomic swap only on success
|
||||
if (newSnapshot.Result.Success)
|
||||
{
|
||||
Interlocked.Exchange(ref _snapshot, newSnapshot);
|
||||
Interlocked.Increment(ref _version);
|
||||
}
|
||||
|
||||
newSnapshot.Result.PreviousVersion = previousVersion;
|
||||
newSnapshot.Result.NewVersion = _version;
|
||||
return newSnapshot.Result;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_reloadLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<PipelineSnapshot> LoadSnapshotAsync(CancellationToken ct)
|
||||
{
|
||||
var result = new PipelineReloadResult();
|
||||
var pipelines = new List<EtlPipelineConfig>();
|
||||
var seenNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Resolve directory path relative to content root
|
||||
var directory = Path.Combine(
|
||||
_environment.ContentRootPath,
|
||||
_options.Value.PipelinesDirectory);
|
||||
|
||||
// Handle missing directory
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
result.Errors.Add(new PipelineLoadError
|
||||
{
|
||||
FileName = directory,
|
||||
ErrorType = "file",
|
||||
Messages = ["Pipeline directory does not exist"]
|
||||
});
|
||||
result.Success = false;
|
||||
return new PipelineSnapshot([], result);
|
||||
}
|
||||
|
||||
// Load each pipeline file
|
||||
var files = Directory.GetFiles(directory, "pipeline.*.json");
|
||||
_logger.LogDebug("Found {Count} pipeline files in {Directory}", files.Length, directory);
|
||||
|
||||
foreach (var file in files)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var fileName = Path.GetFileName(file);
|
||||
EtlPipelineConfig? pipeline = null;
|
||||
|
||||
// Parse JSON
|
||||
try
|
||||
{
|
||||
var json = await File.ReadAllTextAsync(file, ct);
|
||||
pipeline = JsonSerializer.Deserialize<EtlPipelineConfig>(json, PipelineJsonOptions);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
result.Errors.Add(new PipelineLoadError
|
||||
{
|
||||
FileName = fileName,
|
||||
ErrorType = "parse",
|
||||
Messages = [$"JSON parse error: {ex.Message}"]
|
||||
});
|
||||
result.PipelinesSkipped++;
|
||||
continue;
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
result.Errors.Add(new PipelineLoadError
|
||||
{
|
||||
FileName = fileName,
|
||||
ErrorType = "file",
|
||||
Messages = [$"File read error: {ex.Message}"]
|
||||
});
|
||||
result.PipelinesSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pipeline == null)
|
||||
{
|
||||
result.Errors.Add(new PipelineLoadError
|
||||
{
|
||||
FileName = fileName,
|
||||
ErrorType = "parse",
|
||||
Messages = ["Deserialized to null"]
|
||||
});
|
||||
result.PipelinesSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for duplicate names
|
||||
if (!seenNames.Add(pipeline.Name))
|
||||
{
|
||||
result.Errors.Add(new PipelineLoadError
|
||||
{
|
||||
FileName = fileName,
|
||||
PipelineName = pipeline.Name,
|
||||
ErrorType = "validation",
|
||||
Messages = [$"Duplicate pipeline name: {pipeline.Name}"]
|
||||
});
|
||||
result.PipelinesSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate pipeline
|
||||
var validation = _validator.Validate(pipeline, fileName);
|
||||
if (!validation.IsValid)
|
||||
{
|
||||
result.Errors.Add(new PipelineLoadError
|
||||
{
|
||||
FileName = fileName,
|
||||
PipelineName = pipeline.Name,
|
||||
ErrorType = "validation",
|
||||
Messages = validation.Errors
|
||||
});
|
||||
|
||||
// Only skip if enabled (disabled invalid pipelines are warnings)
|
||||
if (pipeline.IsEnabled)
|
||||
{
|
||||
result.PipelinesSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Disabled invalid pipelines are logged as warnings but added
|
||||
_logger.LogWarning(
|
||||
"Disabled pipeline {Name} has validation errors: {Errors}",
|
||||
pipeline.Name,
|
||||
string.Join("; ", validation.Errors));
|
||||
}
|
||||
|
||||
// Log warnings
|
||||
foreach (var warning in validation.Warnings)
|
||||
{
|
||||
_logger.LogWarning("Pipeline {Name}: {Warning}", pipeline.Name, warning);
|
||||
}
|
||||
|
||||
pipelines.Add(pipeline);
|
||||
result.PipelinesLoaded++;
|
||||
}
|
||||
|
||||
// Determine success: all enabled pipelines must be valid
|
||||
// Check if any errors are for enabled pipelines that were skipped
|
||||
var hasEnabledErrors = result.Errors.Any(e =>
|
||||
{
|
||||
// If this error caused a pipeline to be skipped (not in our list),
|
||||
// and it was for an enabled pipeline, it's a failure
|
||||
var matchingPipeline = pipelines.FirstOrDefault(p =>
|
||||
string.Equals(p.Name, e.PipelineName, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
// If pipeline wasn't added to list, it was skipped (enabled + invalid)
|
||||
return matchingPipeline == null && !string.IsNullOrEmpty(e.PipelineName);
|
||||
});
|
||||
|
||||
result.Success = !hasEnabledErrors && result.Errors.All(e => e.ErrorType != "file" || e.Messages.All(m => !m.Contains("does not exist")));
|
||||
|
||||
return new PipelineSnapshot(pipelines, result);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
using JdeScoping.DataSync.Options;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes the pipeline registry at application startup.
|
||||
/// Runs as a hosted service to properly handle async loading.
|
||||
/// </summary>
|
||||
public class PipelineRegistryInitializer : IHostedService
|
||||
{
|
||||
private readonly IPipelineRegistry _registry;
|
||||
private readonly IOptions<DataSyncOptions> _options;
|
||||
private readonly IHostApplicationLifetime _lifetime;
|
||||
private readonly ILogger<PipelineRegistryInitializer> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PipelineRegistryInitializer"/> class.
|
||||
/// </summary>
|
||||
public PipelineRegistryInitializer(
|
||||
IPipelineRegistry registry,
|
||||
IOptions<DataSyncOptions> options,
|
||||
IHostApplicationLifetime lifetime,
|
||||
ILogger<PipelineRegistryInitializer> logger)
|
||||
{
|
||||
_registry = registry ?? throw new ArgumentNullException(nameof(registry));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_lifetime = lifetime ?? throw new ArgumentNullException(nameof(lifetime));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task StartAsync(CancellationToken ct)
|
||||
{
|
||||
_logger.LogInformation("Loading pipeline definitions from {Directory}...",
|
||||
_options.Value.PipelinesDirectory);
|
||||
|
||||
var result = await _registry.ReloadAsync(ct);
|
||||
|
||||
if (!result.Success && _options.Value.StrictPipelineValidation)
|
||||
{
|
||||
_logger.LogCritical(
|
||||
"Pipeline validation failed with {ErrorCount} errors. " +
|
||||
"Application will stop. Set StrictPipelineValidation=false to allow.",
|
||||
result.Errors.Count);
|
||||
|
||||
foreach (var error in result.Errors)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Pipeline {Name} ({File}): [{Type}] {Messages}",
|
||||
error.PipelineName,
|
||||
error.FileName,
|
||||
error.ErrorType,
|
||||
string.Join("; ", error.Messages));
|
||||
}
|
||||
|
||||
_lifetime.StopApplication();
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.Errors.Count > 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Pipeline loading completed with {ErrorCount} warnings",
|
||||
result.Errors.Count);
|
||||
|
||||
foreach (var error in result.Errors)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Pipeline {Name} ({File}): [{Type}] {Messages}",
|
||||
error.PipelineName,
|
||||
error.FileName,
|
||||
error.ErrorType,
|
||||
string.Join("; ", error.Messages));
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Loaded {Count} pipelines ({Enabled} enabled) - version {Version}",
|
||||
result.PipelinesLoaded,
|
||||
_registry.GetEnabledPipelines().Count,
|
||||
result.NewVersion);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task StopAsync(CancellationToken ct) => Task.CompletedTask;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Immutable snapshot of loaded pipelines.
|
||||
/// </summary>
|
||||
internal sealed class PipelineSnapshot
|
||||
{
|
||||
/// <summary>
|
||||
/// Empty snapshot with no pipelines.
|
||||
/// </summary>
|
||||
public static readonly PipelineSnapshot Empty = new([], new PipelineReloadResult { Success = true });
|
||||
|
||||
private readonly Dictionary<string, EtlPipelineConfig> _byName;
|
||||
|
||||
/// <summary>
|
||||
/// Gets all loaded pipelines.
|
||||
/// </summary>
|
||||
public IReadOnlyList<EtlPipelineConfig> AllPipelines { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets only enabled pipelines.
|
||||
/// </summary>
|
||||
public IReadOnlyList<EtlPipelineConfig> EnabledPipelines { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the reload result that created this snapshot.
|
||||
/// </summary>
|
||||
public PipelineReloadResult Result { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the timestamp when this snapshot was loaded.
|
||||
/// </summary>
|
||||
public DateTime LoadedAt { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PipelineSnapshot"/> class.
|
||||
/// </summary>
|
||||
/// <param name="pipelines">The list of pipelines to include.</param>
|
||||
/// <param name="result">The reload result.</param>
|
||||
public PipelineSnapshot(IReadOnlyList<EtlPipelineConfig> pipelines, PipelineReloadResult result)
|
||||
{
|
||||
AllPipelines = pipelines;
|
||||
EnabledPipelines = pipelines.Where(p => p.IsEnabled).ToList();
|
||||
Result = result;
|
||||
LoadedAt = DateTime.UtcNow;
|
||||
_byName = pipelines.ToDictionary(
|
||||
p => p.Name,
|
||||
p => p,
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a pipeline by name (case-insensitive).
|
||||
/// </summary>
|
||||
/// <param name="name">The pipeline name.</param>
|
||||
/// <returns>The pipeline if found, or null.</returns>
|
||||
public EtlPipelineConfig? GetByName(string name) =>
|
||||
_byName.TryGetValue(name, out var p) ? p : null;
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Validates pipeline definitions according to the schema and business rules.
|
||||
/// </summary>
|
||||
public class PipelineValidator : IPipelineValidator
|
||||
{
|
||||
private static readonly HashSet<string> ValidConnections = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"jde", "cms", "giw", "lotfinder"
|
||||
};
|
||||
|
||||
/// <inheritdoc/>
|
||||
public PipelineValidationResult Validate(EtlPipelineConfig pipeline, string fileName)
|
||||
{
|
||||
var result = new PipelineValidationResult { IsValid = true };
|
||||
|
||||
// Extract expected name from filename (pipeline.{Name}.json)
|
||||
var expectedName = ExtractPipelineNameFromFileName(fileName);
|
||||
|
||||
// 1. Name must match filename (case-insensitive)
|
||||
if (!string.Equals(pipeline.Name, expectedName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
result.Errors.Add($"Pipeline name '{pipeline.Name}' does not match filename '{fileName}' (expected '{expectedName}')");
|
||||
}
|
||||
|
||||
// 2. Source is required
|
||||
if (pipeline.Source == null)
|
||||
{
|
||||
result.Errors.Add("Source is required");
|
||||
}
|
||||
else
|
||||
{
|
||||
ValidateSource(pipeline.Source, result);
|
||||
}
|
||||
|
||||
// 3. Destination is required
|
||||
if (pipeline.Destination == null)
|
||||
{
|
||||
result.Errors.Add("Destination is required");
|
||||
}
|
||||
else
|
||||
{
|
||||
ValidateDestination(pipeline.Destination, result);
|
||||
}
|
||||
|
||||
// 4. Interval validation (only for enabled, non-manual-only pipelines)
|
||||
if (pipeline.IsEnabled && !pipeline.IsManualOnly)
|
||||
{
|
||||
ValidateIntervals(pipeline, result);
|
||||
}
|
||||
|
||||
// 5. Script validation
|
||||
ValidateScripts(pipeline.PreScripts, "PreScripts", result);
|
||||
ValidateScripts(pipeline.PostScripts, "PostScripts", result);
|
||||
|
||||
// 6. Warning: MassQuery missing when mass interval set and query has parameters
|
||||
if (pipeline.MassSyncIntervalMinutes.HasValue &&
|
||||
pipeline.Source?.Parameters?.Count > 0 &&
|
||||
string.IsNullOrEmpty(pipeline.Source.MassQuery))
|
||||
{
|
||||
result.Warnings.Add("MassQuery is not specified but pipeline has parameters and mass sync is enabled. Mass sync will use the incremental query which may not work correctly.");
|
||||
}
|
||||
|
||||
// 7. Warning: Hourly without daily
|
||||
if (pipeline.HourlySyncIntervalMinutes.HasValue && !pipeline.DailySyncIntervalMinutes.HasValue)
|
||||
{
|
||||
result.Warnings.Add("HourlySyncIntervalMinutes is set without DailySyncIntervalMinutes. Consider adding daily sync.");
|
||||
}
|
||||
|
||||
result.IsValid = result.Errors.Count == 0;
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string ExtractPipelineNameFromFileName(string fileName)
|
||||
{
|
||||
// Expected format: pipeline.{Name}.json
|
||||
if (fileName.StartsWith("pipeline.", StringComparison.OrdinalIgnoreCase) &&
|
||||
fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return fileName.Substring(9, fileName.Length - 9 - 5); // Remove "pipeline." and ".json"
|
||||
}
|
||||
|
||||
return fileName;
|
||||
}
|
||||
|
||||
private static void ValidateSource(SourceElement source, PipelineValidationResult result)
|
||||
{
|
||||
// Connection must be valid
|
||||
if (string.IsNullOrWhiteSpace(source.Connection))
|
||||
{
|
||||
result.Errors.Add("Source.Connection is required");
|
||||
}
|
||||
else if (!ValidConnections.Contains(source.Connection))
|
||||
{
|
||||
result.Errors.Add($"Source.Connection '{source.Connection}' is not valid. Expected one of: {string.Join(", ", ValidConnections)}");
|
||||
}
|
||||
|
||||
// Query is required
|
||||
if (string.IsNullOrWhiteSpace(source.Query))
|
||||
{
|
||||
result.Errors.Add("Source.Query is required");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateDestination(DestinationElement destination, PipelineValidationResult result)
|
||||
{
|
||||
// Table is required
|
||||
if (string.IsNullOrWhiteSpace(destination.Table))
|
||||
{
|
||||
result.Errors.Add("Destination.Table is required");
|
||||
}
|
||||
|
||||
// MatchColumns must have at least one entry
|
||||
if (destination.MatchColumns == null || destination.MatchColumns.Count == 0)
|
||||
{
|
||||
result.Errors.Add("Destination.MatchColumns must have at least one column");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateIntervals(EtlPipelineConfig pipeline, PipelineValidationResult result)
|
||||
{
|
||||
// At least one interval must be set for enabled, non-manual-only pipelines
|
||||
var hasAnyInterval = pipeline.MassSyncIntervalMinutes.HasValue ||
|
||||
pipeline.DailySyncIntervalMinutes.HasValue ||
|
||||
pipeline.HourlySyncIntervalMinutes.HasValue;
|
||||
|
||||
if (!hasAnyInterval)
|
||||
{
|
||||
result.Errors.Add("At least one sync interval (mass, daily, or hourly) must be set for enabled pipelines. Set IsManualOnly=true for manual-only pipelines.");
|
||||
}
|
||||
|
||||
// All non-null intervals must be positive
|
||||
if (pipeline.MassSyncIntervalMinutes.HasValue && pipeline.MassSyncIntervalMinutes.Value <= 0)
|
||||
{
|
||||
result.Errors.Add("MassSyncIntervalMinutes must be greater than 0");
|
||||
}
|
||||
|
||||
if (pipeline.DailySyncIntervalMinutes.HasValue && pipeline.DailySyncIntervalMinutes.Value <= 0)
|
||||
{
|
||||
result.Errors.Add("DailySyncIntervalMinutes must be greater than 0");
|
||||
}
|
||||
|
||||
if (pipeline.HourlySyncIntervalMinutes.HasValue && pipeline.HourlySyncIntervalMinutes.Value <= 0)
|
||||
{
|
||||
result.Errors.Add("HourlySyncIntervalMinutes must be greater than 0");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateScripts(List<ScriptElement>? scripts, string scriptType, PipelineValidationResult result)
|
||||
{
|
||||
if (scripts == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
for (var i = 0; i < scripts.Count; i++)
|
||||
{
|
||||
var script = scripts[i];
|
||||
if (string.IsNullOrWhiteSpace(script.Script))
|
||||
{
|
||||
result.Errors.Add($"{scriptType}[{i}].Script is required");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using JdeScoping.Core.Models;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.Core.Models.Infrastructure;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Models;
|
||||
@@ -11,10 +12,12 @@ namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Checks schedules and determines which sync tasks need to be executed.
|
||||
/// Uses the pipeline registry for pipeline definitions.
|
||||
/// </summary>
|
||||
public class ScheduleChecker : IScheduleChecker
|
||||
{
|
||||
private readonly IDataUpdateRepository _repository;
|
||||
private readonly IPipelineRegistry _pipelineRegistry;
|
||||
private readonly IOptions<DataSyncOptions> _options;
|
||||
private readonly ILogger<ScheduleChecker> _logger;
|
||||
|
||||
@@ -22,14 +25,17 @@ public class ScheduleChecker : IScheduleChecker
|
||||
/// Initializes a new instance of the <see cref="ScheduleChecker"/> class.
|
||||
/// </summary>
|
||||
/// <param name="repository">Repository for data update records.</param>
|
||||
/// <param name="pipelineRegistry">Registry of pipeline definitions.</param>
|
||||
/// <param name="options">Data sync configuration options.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
public ScheduleChecker(
|
||||
IDataUpdateRepository repository,
|
||||
IPipelineRegistry pipelineRegistry,
|
||||
IOptions<DataSyncOptions> options,
|
||||
ILogger<ScheduleChecker> logger)
|
||||
{
|
||||
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
_pipelineRegistry = pipelineRegistry ?? throw new ArgumentNullException(nameof(pipelineRegistry));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
@@ -41,9 +47,16 @@ public class ScheduleChecker : IScheduleChecker
|
||||
var tasks = new List<DataUpdateTask>();
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
foreach (var config in _options.Value.DataSources.Where(c => c.IsEnabled))
|
||||
// Use the new pipeline registry for scheduling
|
||||
foreach (var pipeline in _pipelineRegistry.GetEnabledPipelines())
|
||||
{
|
||||
var task = CheckConfigSchedule(config, lastUpdates, now);
|
||||
// Skip manual-only pipelines for scheduled execution
|
||||
if (pipeline.IsManualOnly)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var task = CheckPipelineSchedule(pipeline, lastUpdates, now);
|
||||
if (task != null)
|
||||
{
|
||||
tasks.Add(task);
|
||||
@@ -66,63 +79,65 @@ public class ScheduleChecker : IScheduleChecker
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks a single data source config and returns a task if sync is needed.
|
||||
/// Checks a single pipeline and returns a task if sync is needed.
|
||||
/// Priority order: Mass > Daily > Hourly
|
||||
/// </summary>
|
||||
private DataUpdateTask? CheckConfigSchedule(
|
||||
DataSourceConfig config,
|
||||
private DataUpdateTask? CheckPipelineSchedule(
|
||||
EtlPipelineConfig pipeline,
|
||||
Dictionary<string, DataUpdate> lastUpdates,
|
||||
DateTime now)
|
||||
{
|
||||
var tableName = pipeline.Destination.Table;
|
||||
|
||||
// Get last updates for each type
|
||||
var massKey = GetUpdateKey(config.TableName, UpdateTypes.Mass);
|
||||
var dailyKey = GetUpdateKey(config.TableName, UpdateTypes.Daily);
|
||||
var hourlyKey = GetUpdateKey(config.TableName, UpdateTypes.Hourly);
|
||||
var massKey = GetUpdateKey(tableName, UpdateTypes.Mass);
|
||||
var dailyKey = GetUpdateKey(tableName, UpdateTypes.Daily);
|
||||
var hourlyKey = GetUpdateKey(tableName, UpdateTypes.Hourly);
|
||||
|
||||
lastUpdates.TryGetValue(massKey, out var lastMass);
|
||||
lastUpdates.TryGetValue(dailyKey, out var lastDaily);
|
||||
lastUpdates.TryGetValue(hourlyKey, out var lastHourly);
|
||||
|
||||
// Check Mass first (highest priority)
|
||||
if (config.MassConfig.Enabled && NeedsMassSync(config, lastMass, now))
|
||||
if (pipeline.SupportsMassSync && NeedsMassSync(pipeline, lastMass, now))
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Mass sync needed for {Table}: last={LastSync}, interval={Interval}m",
|
||||
config.TableName,
|
||||
tableName,
|
||||
lastMass?.EndDt?.ToString("o") ?? "never",
|
||||
config.MassConfig.IntervalMinutes);
|
||||
pipeline.MassSyncIntervalMinutes);
|
||||
|
||||
return CreateTask(config, UpdateTypes.Mass, null);
|
||||
return DataUpdateTask.FromPipeline(pipeline, UpdateTypes.Mass);
|
||||
}
|
||||
|
||||
// Check Daily
|
||||
if (config.DailyConfig.Enabled && NeedsDailySync(config, lastDaily, lastMass, now))
|
||||
if (pipeline.SupportsDailySync && NeedsDailySync(pipeline, lastDaily, lastMass, now))
|
||||
{
|
||||
var minimumDt = CalculateMinimumDt(lastDaily, config.DailyConfig.IntervalMinutes);
|
||||
var minimumDt = CalculateMinimumDt(lastDaily, pipeline.DailySyncIntervalMinutes!.Value);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Daily sync needed for {Table}: last={LastSync}, interval={Interval}m, minDT={MinDT}",
|
||||
config.TableName,
|
||||
tableName,
|
||||
lastDaily?.EndDt?.ToString("o") ?? "never",
|
||||
config.DailyConfig.IntervalMinutes,
|
||||
pipeline.DailySyncIntervalMinutes,
|
||||
minimumDt?.ToString("o") ?? "null");
|
||||
|
||||
return CreateTask(config, UpdateTypes.Daily, minimumDt);
|
||||
return DataUpdateTask.FromPipeline(pipeline, UpdateTypes.Daily, minimumDt);
|
||||
}
|
||||
|
||||
// Check Hourly
|
||||
if (config.HourlyConfig.Enabled && NeedsHourlySync(config, lastHourly, lastDaily, lastMass, now))
|
||||
if (pipeline.SupportsHourlySync && NeedsHourlySync(pipeline, lastHourly, lastDaily, lastMass, now))
|
||||
{
|
||||
var minimumDt = CalculateMinimumDt(lastHourly, config.HourlyConfig.IntervalMinutes);
|
||||
var minimumDt = CalculateMinimumDt(lastHourly, pipeline.HourlySyncIntervalMinutes!.Value);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Hourly sync needed for {Table}: last={LastSync}, interval={Interval}m, minDT={MinDT}",
|
||||
config.TableName,
|
||||
tableName,
|
||||
lastHourly?.EndDt?.ToString("o") ?? "never",
|
||||
config.HourlyConfig.IntervalMinutes,
|
||||
pipeline.HourlySyncIntervalMinutes,
|
||||
minimumDt?.ToString("o") ?? "null");
|
||||
|
||||
return CreateTask(config, UpdateTypes.Hourly, minimumDt);
|
||||
return DataUpdateTask.FromPipeline(pipeline, UpdateTypes.Hourly, minimumDt);
|
||||
}
|
||||
|
||||
return null;
|
||||
@@ -131,7 +146,7 @@ public class ScheduleChecker : IScheduleChecker
|
||||
/// <summary>
|
||||
/// Determines if a mass sync is needed.
|
||||
/// </summary>
|
||||
private bool NeedsMassSync(DataSourceConfig config, DataUpdate? lastMass, DateTime now)
|
||||
private static bool NeedsMassSync(EtlPipelineConfig pipeline, DataUpdate? lastMass, DateTime now)
|
||||
{
|
||||
// Never synced before - need mass sync
|
||||
if (lastMass == null)
|
||||
@@ -147,14 +162,18 @@ public class ScheduleChecker : IScheduleChecker
|
||||
}
|
||||
|
||||
// EndDt is set for successful syncs (GetLastDataUpdatesAsync filters WasSuccessful=1)
|
||||
var nextSyncDue = lastMass.EndDt!.Value.AddMinutes(config.MassConfig.IntervalMinutes);
|
||||
var nextSyncDue = lastMass.EndDt!.Value.AddMinutes(pipeline.MassSyncIntervalMinutes!.Value);
|
||||
return now > nextSyncDue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines if a daily sync is needed.
|
||||
/// </summary>
|
||||
private bool NeedsDailySync(DataSourceConfig config, DataUpdate? lastDaily, DataUpdate? lastMass, DateTime now)
|
||||
private static bool NeedsDailySync(
|
||||
EtlPipelineConfig pipeline,
|
||||
DataUpdate? lastDaily,
|
||||
DataUpdate? lastMass,
|
||||
DateTime now)
|
||||
{
|
||||
// If no mass sync ever happened, we need mass first
|
||||
if (lastMass == null)
|
||||
@@ -175,15 +194,15 @@ public class ScheduleChecker : IScheduleChecker
|
||||
}
|
||||
|
||||
// EndDt is set for successful syncs (GetLastDataUpdatesAsync filters WasSuccessful=1)
|
||||
var nextSyncDue = lastDaily.EndDt!.Value.AddMinutes(config.DailyConfig.IntervalMinutes);
|
||||
var nextSyncDue = lastDaily.EndDt!.Value.AddMinutes(pipeline.DailySyncIntervalMinutes!.Value);
|
||||
return now > nextSyncDue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines if an hourly sync is needed.
|
||||
/// </summary>
|
||||
private bool NeedsHourlySync(
|
||||
DataSourceConfig config,
|
||||
private static bool NeedsHourlySync(
|
||||
EtlPipelineConfig pipeline,
|
||||
DataUpdate? lastHourly,
|
||||
DataUpdate? lastDaily,
|
||||
DataUpdate? lastMass,
|
||||
@@ -208,7 +227,7 @@ public class ScheduleChecker : IScheduleChecker
|
||||
}
|
||||
|
||||
// EndDt is set for successful syncs (GetLastDataUpdatesAsync filters WasSuccessful=1)
|
||||
var nextSyncDue = lastHourly.EndDt!.Value.AddMinutes(config.HourlyConfig.IntervalMinutes);
|
||||
var nextSyncDue = lastHourly.EndDt!.Value.AddMinutes(pipeline.HourlySyncIntervalMinutes!.Value);
|
||||
return now > nextSyncDue;
|
||||
}
|
||||
|
||||
@@ -227,22 +246,6 @@ public class ScheduleChecker : IScheduleChecker
|
||||
return lastUpdate.EndDt!.Value.AddMinutes(-lookbackMinutes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a data update task.
|
||||
/// </summary>
|
||||
private static DataUpdateTask CreateTask(DataSourceConfig config, UpdateTypes updateType, DateTime? minimumDt)
|
||||
{
|
||||
return new DataUpdateTask
|
||||
{
|
||||
TableName = config.TableName,
|
||||
SourceSystem = config.SourceSystem,
|
||||
SourceData = config.SourceData,
|
||||
UpdateType = updateType,
|
||||
MinimumDt = minimumDt,
|
||||
Config = config
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the dictionary key for looking up last updates.
|
||||
/// </summary>
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Models;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Telemetry;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -106,4 +107,53 @@ public class SyncOrchestrator : ISyncOrchestrator
|
||||
|
||||
_metrics.RecordCycleCompleted(completedCount, failedCount, elapsed.TotalSeconds);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task ExecuteSingleSyncAsync(DataUpdateTask task, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Executing single sync for {Table} ({Type})",
|
||||
task.TableName,
|
||||
task.UpdateType);
|
||||
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
await using var scope = _scopeFactory.CreateAsyncScope();
|
||||
|
||||
try
|
||||
{
|
||||
var operation = scope.ServiceProvider.GetRequiredService<ITableSyncOperation>();
|
||||
await operation.ExecuteAsync(task, cancellationToken);
|
||||
|
||||
var elapsed = DateTime.UtcNow - startTime;
|
||||
_logger.LogInformation(
|
||||
"Single sync for {Table} ({Type}) completed in {Elapsed:F1}s",
|
||||
task.TableName,
|
||||
task.UpdateType,
|
||||
elapsed.TotalSeconds);
|
||||
|
||||
_metrics.RecordCycleCompleted(1, 0, elapsed.TotalSeconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Single sync for {Table} ({Type}) was cancelled",
|
||||
task.TableName,
|
||||
task.UpdateType);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var elapsed = DateTime.UtcNow - startTime;
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Single sync for {Table} ({Type}) failed after {Elapsed:F1}s",
|
||||
task.TableName,
|
||||
task.UpdateType,
|
||||
elapsed.TotalSeconds);
|
||||
|
||||
_metrics.RecordCycleCompleted(0, 1, elapsed.TotalSeconds);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
using JdeScoping.Core.Interfaces;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataAccess.Services;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Models;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using JdeScoping.DataSync.Telemetry;
|
||||
using JdeScoping.Domain.Models;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -11,7 +16,7 @@ namespace JdeScoping.DataSync;
|
||||
|
||||
/// <summary>
|
||||
/// Unified background service that coordinates data synchronization and search processing.
|
||||
/// Data freshness takes priority over search processing.
|
||||
/// Priority order: Manual sync requests > Scheduled syncs > Search processing.
|
||||
/// </summary>
|
||||
public class WorkProcessor : BackgroundService
|
||||
{
|
||||
@@ -52,8 +57,9 @@ public class WorkProcessor : BackgroundService
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"WorkProcessor starting with WorkInterval={WorkInterval}",
|
||||
_options.WorkInterval);
|
||||
"WorkProcessor starting with WorkInterval={WorkInterval}, MaxManualRequestsPerCycle={MaxManual}",
|
||||
_options.WorkInterval,
|
||||
_options.MaxManualRequestsPerCycle);
|
||||
|
||||
// Startup cleanup
|
||||
await StartupCleanupAsync(stoppingToken);
|
||||
@@ -135,13 +141,21 @@ public class WorkProcessor : BackgroundService
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs one work cycle: data syncs have priority, then search processing.
|
||||
/// Performs one work cycle: manual syncs (priority 1), scheduled syncs (priority 2), search processing (priority 3).
|
||||
/// </summary>
|
||||
private async Task<string> DoWorkAsync(CancellationToken ct)
|
||||
{
|
||||
await using var scope = _scopeFactory.CreateAsyncScope();
|
||||
|
||||
// Priority 1: Data syncs
|
||||
// Priority 1: Manual sync requests (with fairness cap)
|
||||
var processedManualCount = await ProcessManualSyncRequestsAsync(scope, ct);
|
||||
if (processedManualCount > 0)
|
||||
{
|
||||
// After processing manual requests, still check scheduled syncs
|
||||
// This implements the fairness policy - manual requests don't completely block scheduled syncs
|
||||
}
|
||||
|
||||
// Priority 2: Data syncs
|
||||
var scheduleChecker = scope.ServiceProvider.GetRequiredService<IScheduleChecker>();
|
||||
var pendingTasks = await scheduleChecker.GetPendingTasksAsync(ct);
|
||||
|
||||
@@ -158,7 +172,7 @@ public class WorkProcessor : BackgroundService
|
||||
return "Idle";
|
||||
}
|
||||
|
||||
// Priority 2: Search processing (only when syncs are current)
|
||||
// Priority 3: Search processing (only when syncs are current)
|
||||
var searchRepository = scope.ServiceProvider.GetRequiredService<ISearchRepository>();
|
||||
var search = await searchRepository.GetNextQueuedSearchAsync(ct);
|
||||
|
||||
@@ -176,6 +190,161 @@ public class WorkProcessor : BackgroundService
|
||||
return "Idle";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes pending manual sync requests up to the configured limit.
|
||||
/// </summary>
|
||||
/// <returns>The number of manual requests processed.</returns>
|
||||
private async Task<int> ProcessManualSyncRequestsAsync(AsyncServiceScope scope, CancellationToken ct)
|
||||
{
|
||||
var manualSyncService = scope.ServiceProvider.GetService<IManualSyncRequestService>();
|
||||
if (manualSyncService == null)
|
||||
{
|
||||
// Service not registered - skip manual sync processing
|
||||
return 0;
|
||||
}
|
||||
|
||||
var pipelineRegistry = scope.ServiceProvider.GetRequiredService<IPipelineRegistry>();
|
||||
var orchestrator = scope.ServiceProvider.GetRequiredService<ISyncOrchestrator>();
|
||||
|
||||
var processedCount = 0;
|
||||
var maxRequests = _options.MaxManualRequestsPerCycle > 0
|
||||
? _options.MaxManualRequestsPerCycle
|
||||
: int.MaxValue;
|
||||
|
||||
while (processedCount < maxRequests)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var request = await manualSyncService.GetNextPendingRequestAsync(ct);
|
||||
if (request == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await ProcessManualSyncRequestAsync(
|
||||
request,
|
||||
pipelineRegistry,
|
||||
orchestrator,
|
||||
manualSyncService,
|
||||
ct);
|
||||
|
||||
processedCount++;
|
||||
}
|
||||
|
||||
if (processedCount > 0)
|
||||
{
|
||||
_logger.LogInformation("Processed {Count} manual sync requests", processedCount);
|
||||
}
|
||||
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes a single manual sync request.
|
||||
/// </summary>
|
||||
private async Task ProcessManualSyncRequestAsync(
|
||||
ManualSyncRequest request,
|
||||
IPipelineRegistry pipelineRegistry,
|
||||
ISyncOrchestrator orchestrator,
|
||||
IManualSyncRequestService manualSyncService,
|
||||
CancellationToken ct)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Processing manual sync request #{Id}: Pipeline={Pipeline}, SyncType={SyncType}",
|
||||
request.Id, request.PipelineName, request.SyncType);
|
||||
|
||||
await NotifyStatusSafeAsync($"Processing manual sync: {request.PipelineName}", ct);
|
||||
|
||||
try
|
||||
{
|
||||
// Get pipeline from registry
|
||||
var pipeline = pipelineRegistry.GetPipeline(request.PipelineName);
|
||||
if (pipeline == null)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Manual sync request #{Id} failed: Pipeline '{Name}' not found",
|
||||
request.Id, request.PipelineName);
|
||||
// Can't mark as failed without adding a method - just complete it
|
||||
await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!pipeline.IsEnabled)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Manual sync request #{Id} failed: Pipeline '{Name}' is disabled",
|
||||
request.Id, request.PipelineName);
|
||||
await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse sync type
|
||||
if (!TryParseUpdateType(request.SyncType, out var updateType))
|
||||
{
|
||||
_logger.LogError(
|
||||
"Manual sync request #{Id} failed: Invalid sync type '{Type}'",
|
||||
request.Id, request.SyncType);
|
||||
await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate that the pipeline supports this sync type
|
||||
if (!pipelineRegistry.IsValidPipelineAndSyncType(request.PipelineName, request.SyncType))
|
||||
{
|
||||
_logger.LogError(
|
||||
"Manual sync request #{Id} failed: Pipeline '{Name}' does not support {Type} sync",
|
||||
request.Id, request.PipelineName, request.SyncType);
|
||||
await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
// Create and execute the sync task
|
||||
var task = DataUpdateTask.FromPipeline(pipeline, updateType);
|
||||
await orchestrator.ExecuteSingleSyncAsync(task, ct);
|
||||
|
||||
// Mark request as completed
|
||||
await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Manual sync request #{Id} completed successfully",
|
||||
request.Id);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"Manual sync request #{Id} failed with error",
|
||||
request.Id);
|
||||
|
||||
// Try to mark as completed (best effort)
|
||||
try
|
||||
{
|
||||
await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct);
|
||||
}
|
||||
catch (Exception completeEx)
|
||||
{
|
||||
_logger.LogWarning(completeEx,
|
||||
"Failed to mark manual sync request #{Id} as completed after error",
|
||||
request.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tries to parse a sync type string to an UpdateTypes enum value.
|
||||
/// </summary>
|
||||
private static bool TryParseUpdateType(string syncType, out UpdateTypes updateType)
|
||||
{
|
||||
updateType = syncType.ToLowerInvariant() switch
|
||||
{
|
||||
"mass" => UpdateTypes.Mass,
|
||||
"daily" => UpdateTypes.Daily,
|
||||
"hourly" => UpdateTypes.Hourly,
|
||||
_ => (UpdateTypes)(-1)
|
||||
};
|
||||
|
||||
return (int)updateType >= 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Purges old DataUpdate entries periodically (every 24 hours).
|
||||
/// </summary>
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
-- Migration: 060_CreateManualSyncRequestTable
|
||||
-- Purpose: Create ManualSyncRequest table for storing manually requested data syncs
|
||||
|
||||
SET QUOTED_IDENTIFIER ON;
|
||||
SET ANSI_NULLS ON;
|
||||
GO
|
||||
|
||||
-- Create ManualSyncRequest table
|
||||
IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'ManualSyncRequest' AND schema_id = SCHEMA_ID('dbo'))
|
||||
BEGIN
|
||||
CREATE TABLE [dbo].[ManualSyncRequest]
|
||||
(
|
||||
[ID] INT IDENTITY(1,1) NOT NULL,
|
||||
[PipelineName] VARCHAR(128) NOT NULL,
|
||||
[SyncType] VARCHAR(16) NOT NULL,
|
||||
[RequestDT] DATETIME2(7) NOT NULL CONSTRAINT [DF_ManualSyncRequest_RequestDT] DEFAULT (SYSUTCDATETIME()),
|
||||
[RequestedBy] VARCHAR(128) NOT NULL,
|
||||
[CompletedDT] DATETIME2(7) NULL,
|
||||
[CancelDT] DATETIME2(7) NULL,
|
||||
[CancelledBy] VARCHAR(128) NULL,
|
||||
[RowVersion] ROWVERSION NOT NULL,
|
||||
CONSTRAINT [PK_ManualSyncRequest] PRIMARY KEY CLUSTERED ([ID]),
|
||||
CONSTRAINT [CK_ManualSyncRequest_SyncType] CHECK ([SyncType] IN ('mass', 'daily', 'hourly'))
|
||||
);
|
||||
END
|
||||
GO
|
||||
|
||||
-- Index for pending request queries (FIFO ordering)
|
||||
IF NOT EXISTS (SELECT * FROM sys.indexes WHERE name = 'IX_ManualSyncRequest_Pending' AND object_id = OBJECT_ID('dbo.ManualSyncRequest'))
|
||||
BEGIN
|
||||
CREATE NONCLUSTERED INDEX [IX_ManualSyncRequest_Pending]
|
||||
ON [dbo].[ManualSyncRequest] ([CompletedDT], [CancelDT], [RequestDT])
|
||||
WHERE [CompletedDT] IS NULL AND [CancelDT] IS NULL;
|
||||
END
|
||||
GO
|
||||
|
||||
-- Index for user-specific queries
|
||||
IF NOT EXISTS (SELECT * FROM sys.indexes WHERE name = 'IX_ManualSyncRequest_RequestedBy' AND object_id = OBJECT_ID('dbo.ManualSyncRequest'))
|
||||
BEGIN
|
||||
CREATE NONCLUSTERED INDEX [IX_ManualSyncRequest_RequestedBy]
|
||||
ON [dbo].[ManualSyncRequest] ([RequestedBy], [RequestDT] DESC);
|
||||
END
|
||||
GO
|
||||
@@ -0,0 +1,9 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,59 @@
|
||||
namespace JdeScoping.Domain.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a manually requested data sync operation.
|
||||
/// </summary>
|
||||
public class ManualSyncRequest
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the unique identifier.
|
||||
/// </summary>
|
||||
public int Id { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the name of the ETL pipeline to sync.
|
||||
/// </summary>
|
||||
public string PipelineName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the type of sync (mass, daily, hourly).
|
||||
/// </summary>
|
||||
public string SyncType { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the UTC timestamp when the request was created.
|
||||
/// </summary>
|
||||
public DateTime RequestDT { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the username of the user who created the request.
|
||||
/// </summary>
|
||||
public string RequestedBy { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the UTC timestamp when the sync completed, or null if pending.
|
||||
/// </summary>
|
||||
public DateTime? CompletedDT { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the UTC timestamp when the request was cancelled, or null if not cancelled.
|
||||
/// </summary>
|
||||
public DateTime? CancelDT { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the username of the user who cancelled the request, or null.
|
||||
/// </summary>
|
||||
public string? CancelledBy { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the row version for optimistic concurrency.
|
||||
/// </summary>
|
||||
public byte[] RowVersion { get; set; } = Array.Empty<byte>();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the derived status based on CompletedDT and CancelDT.
|
||||
/// </summary>
|
||||
public string Status => CancelDT.HasValue ? "Cancelled"
|
||||
: CompletedDT.HasValue ? "Completed"
|
||||
: "Pending";
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "Branch",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "Branch",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "FunctionCode",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code",
|
||||
"massQuery": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code",
|
||||
"parameters": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "FunctionCode",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "Item",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL AND (pn.IMUPMJ > :dateUpdated OR (pn.IMUPMJ = :dateUpdated AND pn.IMTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "Item",
|
||||
"matchColumns": ["ShortItemNumber"],
|
||||
"excludeFromUpdate": ["ShortItemNumber", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "JdeUser",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1",
|
||||
"massQuery": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1",
|
||||
"parameters": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "JdeUser",
|
||||
"matchColumns": ["AddressNumber"],
|
||||
"excludeFromUpdate": ["AddressNumber", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "Lot",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL AND (lot.IOUPMJ > :dateUpdated OR (lot.IOUPMJ = :dateUpdated AND lot.IOTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "Lot",
|
||||
"matchColumns": ["LotNumber", "BranchCode"],
|
||||
"excludeFromUpdate": ["LotNumber", "BranchCode", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "LotUsage_Curr",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL AND (lu.ILTRDJ > :dateUpdated OR (lu.ILTRDJ = :dateUpdated AND lu.ILTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "LotUsage_Curr",
|
||||
"matchColumns": ["UniqueId"],
|
||||
"excludeFromUpdate": ["UniqueId", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "MisData_Curr",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 100800,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": null,
|
||||
"source": {
|
||||
"connection": "cms",
|
||||
"query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current' AND Status.PDATE_RELEASED >= :lastUpdateDT",
|
||||
"massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current'",
|
||||
"parameters": {
|
||||
"lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "MisData_Curr",
|
||||
"matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"],
|
||||
"excludeFromUpdate": []
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": [
|
||||
{ "connection": "lotfinder", "script": "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Curr.MisNumber = bl.MisNumber AND MisData_Curr.RevID = bl.RevID AND MisData_Curr.Status = 'Current' AND bl.Status = 'BackLevel';" },
|
||||
{ "connection": "lotfinder", "script": "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Curr.MisNumber = nl.MisNumber AND MisData_Curr.RevID < nl.RevID AND MisData_Curr.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;" },
|
||||
{ "connection": "lotfinder", "script": "MERGE INTO dbo.MisData_Hist AS target USING (SELECT * FROM dbo.MisData_Curr WHERE Status = 'BackLevel') AS source ON target.ItemNumber = source.ItemNumber AND target.BranchCode = source.BranchCode AND target.SequenceNumber = source.SequenceNumber AND target.MisNumber = source.MisNumber AND target.CharNumber = source.CharNumber WHEN MATCHED THEN UPDATE SET target.RevID = source.RevID, target.TestDescription = source.TestDescription, target.SamplingType = source.SamplingType, target.SamplingValue = source.SamplingValue, target.ToolsGauges = source.ToolsGauges, target.WorkInstructions = source.WorkInstructions, target.Status = source.Status, target.ReleaseDate = source.ReleaseDate, target.ObsoleteDate = source.ObsoleteDate WHEN NOT MATCHED THEN INSERT (ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, Status, ReleaseDate, ObsoleteDate) VALUES (source.ItemNumber, source.BranchCode, source.SequenceNumber, source.MisNumber, source.RevID, source.CharNumber, source.TestDescription, source.SamplingType, source.SamplingValue, source.ToolsGauges, source.WorkInstructions, source.Status, source.ReleaseDate, source.ObsoleteDate);" },
|
||||
{ "connection": "lotfinder", "script": "DELETE FROM dbo.MisData_Curr WHERE Status = 'BackLevel';" },
|
||||
{ "connection": "lotfinder", "script": "ALTER INDEX [PK_MisData_Curr] ON [dbo].[MisData_Curr] REBUILD;" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "MisData_Hist",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 100800,
|
||||
"dailySyncIntervalMinutes": null,
|
||||
"hourlySyncIntervalMinutes": null,
|
||||
"source": {
|
||||
"connection": "cms",
|
||||
"query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel' AND Status.PDATE_RELEASED >= :lastUpdateDT",
|
||||
"massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel'",
|
||||
"parameters": {
|
||||
"lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "MisData_Hist",
|
||||
"matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"],
|
||||
"excludeFromUpdate": []
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": [
|
||||
{ "connection": "lotfinder", "script": "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Hist.MisNumber = bl.MisNumber AND MisData_Hist.RevID = bl.RevID AND MisData_Hist.Status = 'Current' AND bl.Status = 'BackLevel';" },
|
||||
{ "connection": "lotfinder", "script": "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Hist.MisNumber = nl.MisNumber AND MisData_Hist.RevID < nl.RevID AND MisData_Hist.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;" },
|
||||
{ "connection": "lotfinder", "script": "ALTER INDEX [PK_MisData_Hist] ON [dbo].[MisData_Hist] REBUILD;" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "OrgHierarchy",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL AND (oh.DATEUPDATED_IWUPMJ > :dateUpdated OR (oh.DATEUPDATED_IWUPMJ = :dateUpdated AND oh.TIMEOFDAY_IWTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "OrgHierarchy",
|
||||
"matchColumns": ["WorkCenterCode", "BranchCode"],
|
||||
"excludeFromUpdate": ["WorkCenterCode", "BranchCode", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "ProfitCenter",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "ProfitCenter",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "RouteMaster",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL AND (route_master.DATEUPDATED_IRUPMJ > :dateUpdated OR (route_master.DATEUPDATED_IRUPMJ = :dateUpdated AND route_master.TIMEOFDAY_IRTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "RouteMaster",
|
||||
"matchColumns": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber"],
|
||||
"excludeFromUpdate": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "StatusCode",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "giw",
|
||||
"query": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL AND (sc.DATEUPDATED_DRUPMJ > :dateUpdated OR (sc.DATEUPDATED_DRUPMJ = :dateUpdated AND sc.TIMELASTUPDATED_DRUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "StatusCode",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "WorkCenter",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkCenter",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "WorkOrderComponent_Curr",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL AND (woc.DATEUPDATED_WMUPMJ > :dateUpdated OR (woc.DATEUPDATED_WMUPMJ = :dateUpdated AND woc.TIMEOFDAY_WMTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderComponent_Curr",
|
||||
"matchColumns": ["UniqueID"],
|
||||
"excludeFromUpdate": ["UniqueID", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "WorkOrderRouting",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410' AND (woz.DATEUPDATED_SZUPMJ > :dateUpdated OR (woz.DATEUPDATED_SZUPMJ = :dateUpdated AND woz.TIMEOFDAY_SZTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderRouting",
|
||||
"matchColumns": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber"],
|
||||
"excludeFromUpdate": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "WorkOrderStep_Curr",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY) WHERE (wos.DATEUPDATED_WLUPMJ > :dateUpdated OR (wos.DATEUPDATED_WLUPMJ = :dateUpdated AND wos.TIMEOFDAY_WLTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY)",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderStep_Curr",
|
||||
"matchColumns": ["WorkOrderNumber", "BranchCode", "StepNumber"],
|
||||
"excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "StepNumber", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "WorkOrderTime_Curr",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot WHERE (wot.DATEUPDATED_WTUPMJ > :dateUpdated OR (wot.DATEUPDATED_WTUPMJ = :dateUpdated AND wot.TIMEOFDAY_WTTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderTime_Curr",
|
||||
"matchColumns": ["UniqueID"],
|
||||
"excludeFromUpdate": ["UniqueID", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "WorkOrder_Curr",
|
||||
"isEnabled": true,
|
||||
"isManualOnly": false,
|
||||
"massSyncIntervalMinutes": 10080,
|
||||
"dailySyncIntervalMinutes": 1440,
|
||||
"hourlySyncIntervalMinutes": 60,
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo WHERE (wo.WAUPMJ > :dateUpdated OR (wo.WAUPMJ = :dateUpdated AND wo.WATDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrder_Curr",
|
||||
"matchColumns": ["WorkOrderNumber", "BranchCode"],
|
||||
"excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "LastUpdateDt"]
|
||||
},
|
||||
"preScripts": [],
|
||||
"postScripts": []
|
||||
}
|
||||
@@ -11,6 +11,8 @@
|
||||
},
|
||||
"DataSync": {
|
||||
"Enabled": true,
|
||||
"PipelinesDirectory": "Pipelines",
|
||||
"StrictPipelineValidation": true,
|
||||
"CheckInterval": "00:01:00",
|
||||
"MaxDegreeOfParallelism": 8,
|
||||
"BatchSize": 1000000,
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration section for connection strings.
|
||||
/// Supports both the standard .NET dictionary format and structured entries list.
|
||||
/// </summary>
|
||||
[JsonConverter(typeof(ConnectionStringsSectionConverter))]
|
||||
public class ConnectionStringsSection
|
||||
{
|
||||
public List<ConnectionStringEntry> Entries { get; set; } = new();
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Custom JSON converter that handles the standard .NET ConnectionStrings dictionary format
|
||||
/// and converts it to a ConnectionStringsSection with an Entries list.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Standard appsettings.json uses a dictionary format:
|
||||
/// <code>
|
||||
/// "ConnectionStrings": {
|
||||
/// "LotFinder": "Server=localhost;Database=...;",
|
||||
/// "JDE": "Data Source=...;"
|
||||
/// }
|
||||
/// </code>
|
||||
/// This converter parses that into a list of ConnectionStringEntry objects.
|
||||
/// </remarks>
|
||||
public class ConnectionStringsSectionConverter : JsonConverter<ConnectionStringsSection>
|
||||
{
|
||||
public override ConnectionStringsSection? Read(
|
||||
ref Utf8JsonReader reader,
|
||||
Type typeToConvert,
|
||||
JsonSerializerOptions options)
|
||||
{
|
||||
var section = new ConnectionStringsSection();
|
||||
|
||||
if (reader.TokenType != JsonTokenType.StartObject)
|
||||
{
|
||||
return section;
|
||||
}
|
||||
|
||||
while (reader.Read())
|
||||
{
|
||||
if (reader.TokenType == JsonTokenType.EndObject)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (reader.TokenType == JsonTokenType.PropertyName)
|
||||
{
|
||||
var propertyName = reader.GetString();
|
||||
reader.Read();
|
||||
|
||||
// Check if this is the "Entries" property (our internal format)
|
||||
if (string.Equals(propertyName, "entries", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Read as array of ConnectionStringEntry
|
||||
if (reader.TokenType == JsonTokenType.StartArray)
|
||||
{
|
||||
section.Entries = JsonSerializer.Deserialize<List<ConnectionStringEntry>>(ref reader, options)
|
||||
?? new List<ConnectionStringEntry>();
|
||||
}
|
||||
}
|
||||
else if (!string.IsNullOrEmpty(propertyName))
|
||||
{
|
||||
// Standard dictionary format: property name is the connection name,
|
||||
// value is the connection string
|
||||
var connectionString = reader.TokenType == JsonTokenType.String
|
||||
? reader.GetString()
|
||||
: null;
|
||||
|
||||
if (!string.IsNullOrEmpty(connectionString))
|
||||
{
|
||||
var entry = ParseConnectionString(propertyName, connectionString);
|
||||
section.Entries.Add(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return section;
|
||||
}
|
||||
|
||||
public override void Write(
|
||||
Utf8JsonWriter writer,
|
||||
ConnectionStringsSection value,
|
||||
JsonSerializerOptions options)
|
||||
{
|
||||
// Write back in standard dictionary format for compatibility
|
||||
writer.WriteStartObject();
|
||||
|
||||
foreach (var entry in value.Entries)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(entry.Name))
|
||||
{
|
||||
writer.WriteString(entry.Name, entry.GenerateConnectionString());
|
||||
}
|
||||
}
|
||||
|
||||
writer.WriteEndObject();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses a connection string and attempts to detect the provider type.
|
||||
/// </summary>
|
||||
private static ConnectionStringEntry ParseConnectionString(string name, string connectionString)
|
||||
{
|
||||
var entry = new ConnectionStringEntry
|
||||
{
|
||||
Name = name,
|
||||
RawConnectionString = connectionString
|
||||
};
|
||||
|
||||
// Try to detect provider and parse structured fields
|
||||
var parts = ParseConnectionStringParts(connectionString);
|
||||
|
||||
// Detect Oracle first (Data Source with host:port/service pattern)
|
||||
if (parts.TryGetValue("data source", out var dataSource) &&
|
||||
IsOracleDataSource(dataSource))
|
||||
{
|
||||
entry.Provider = ConnectionProvider.Oracle;
|
||||
ParseOracleDataSource(entry, dataSource);
|
||||
|
||||
if (parts.TryGetValue("user id", out var oraUserId))
|
||||
{
|
||||
entry.UserId = oraUserId;
|
||||
}
|
||||
|
||||
if (parts.TryGetValue("password", out var oraPassword))
|
||||
{
|
||||
entry.Password = oraPassword;
|
||||
}
|
||||
}
|
||||
// Detect SQL Server (Server property or Data Source without Oracle pattern)
|
||||
else if (parts.TryGetValue("server", out var server) ||
|
||||
parts.TryGetValue("data source", out server))
|
||||
{
|
||||
entry.Provider = ConnectionProvider.SqlServer;
|
||||
entry.Server = server;
|
||||
|
||||
if (parts.TryGetValue("database", out var database) ||
|
||||
parts.TryGetValue("initial catalog", out database))
|
||||
{
|
||||
entry.Database = database;
|
||||
}
|
||||
|
||||
if (parts.TryGetValue("user id", out var userId) ||
|
||||
parts.TryGetValue("uid", out userId))
|
||||
{
|
||||
entry.UserId = userId;
|
||||
}
|
||||
|
||||
if (parts.TryGetValue("password", out var password) ||
|
||||
parts.TryGetValue("pwd", out password))
|
||||
{
|
||||
entry.Password = password;
|
||||
}
|
||||
|
||||
if (parts.TryGetValue("encrypt", out var encrypt))
|
||||
{
|
||||
entry.Encrypt = encrypt;
|
||||
}
|
||||
|
||||
if (parts.TryGetValue("trustservercertificate", out var trustCert) ||
|
||||
parts.TryGetValue("trust server certificate", out trustCert))
|
||||
{
|
||||
entry.TrustServerCertificate = trustCert.Equals("true", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
if (parts.TryGetValue("connection timeout", out var timeout) ||
|
||||
parts.TryGetValue("connect timeout", out timeout))
|
||||
{
|
||||
if (int.TryParse(timeout, out var timeoutValue))
|
||||
{
|
||||
entry.ConnectionTimeout = timeoutValue;
|
||||
}
|
||||
}
|
||||
|
||||
if (parts.TryGetValue("application name", out var appName))
|
||||
{
|
||||
entry.ApplicationName = appName;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Generic/unknown provider - just store raw connection string
|
||||
entry.Provider = ConnectionProvider.Generic;
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses a connection string into key-value pairs.
|
||||
/// </summary>
|
||||
private static Dictionary<string, string> ParseConnectionStringParts(string connectionString)
|
||||
{
|
||||
var parts = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Split by semicolon, handling quoted values
|
||||
var segments = connectionString.Split(';', StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
foreach (var segment in segments)
|
||||
{
|
||||
var equalsIndex = segment.IndexOf('=');
|
||||
if (equalsIndex > 0)
|
||||
{
|
||||
var key = segment[..equalsIndex].Trim();
|
||||
var value = segment[(equalsIndex + 1)..].Trim();
|
||||
parts[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return parts;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines if a Data Source value looks like an Oracle connection string.
|
||||
/// Oracle typically uses host:port/service format with numeric port and service name after slash.
|
||||
/// SQL Server uses server,port or server\instance format.
|
||||
/// </summary>
|
||||
private static bool IsOracleDataSource(string dataSource)
|
||||
{
|
||||
// Check for Oracle patterns: //host:port/service or host:port/service
|
||||
// SQL Server patterns: server,port or server\instance or just server
|
||||
|
||||
if (string.IsNullOrWhiteSpace(dataSource))
|
||||
return false;
|
||||
|
||||
var source = dataSource.TrimStart('/');
|
||||
|
||||
// Oracle format: has colon followed by port number and then slash
|
||||
// e.g., "jde-server:1521/JDEPROD" or "//db-host:1523/PRODDB"
|
||||
var colonIndex = source.IndexOf(':');
|
||||
var slashIndex = source.IndexOf('/');
|
||||
|
||||
if (colonIndex > 0 && slashIndex > colonIndex)
|
||||
{
|
||||
// Check if what's between colon and slash is a numeric port
|
||||
var potentialPort = source[(colonIndex + 1)..slashIndex];
|
||||
if (int.TryParse(potentialPort, out _))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Also check for //host pattern without port
|
||||
if (dataSource.StartsWith("//"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses Oracle Data Source format (e.g., "//host:port/service" or "host:port/service").
|
||||
/// </summary>
|
||||
private static void ParseOracleDataSource(ConnectionStringEntry entry, string dataSource)
|
||||
{
|
||||
// Remove leading slashes if present
|
||||
var source = dataSource.TrimStart('/');
|
||||
|
||||
// Try to parse host:port/service format
|
||||
var colonIndex = source.IndexOf(':');
|
||||
var slashIndex = source.IndexOf('/');
|
||||
|
||||
if (colonIndex > 0)
|
||||
{
|
||||
entry.Host = source[..colonIndex];
|
||||
|
||||
if (slashIndex > colonIndex)
|
||||
{
|
||||
var portStr = source[(colonIndex + 1)..slashIndex];
|
||||
if (int.TryParse(portStr, out var port))
|
||||
{
|
||||
entry.Port = port;
|
||||
}
|
||||
|
||||
entry.ServiceName = source[(slashIndex + 1)..];
|
||||
}
|
||||
else
|
||||
{
|
||||
var portStr = source[(colonIndex + 1)..];
|
||||
if (int.TryParse(portStr, out var port))
|
||||
{
|
||||
entry.Port = port;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (slashIndex > 0)
|
||||
{
|
||||
entry.Host = source[..slashIndex];
|
||||
entry.ServiceName = source[(slashIndex + 1)..];
|
||||
}
|
||||
else
|
||||
{
|
||||
entry.Host = source;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.ConfigManager.Services;
|
||||
using JdeScoping.ConfigManager.ViewModels.PipelineSteps;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Windows.Input;
|
||||
@@ -12,14 +13,16 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
{
|
||||
private readonly PipelineModel _model;
|
||||
private readonly Action _onChanged;
|
||||
private readonly IDialogService _dialogService;
|
||||
private PipelineStepViewModelBase? _selectedStep;
|
||||
private object? _selectedStepEditor;
|
||||
|
||||
public PipelineEditorViewModel(string name, PipelineModel model, IReadOnlyList<string> availableConnections, Action onChanged)
|
||||
public PipelineEditorViewModel(string name, PipelineModel model, IReadOnlyList<string> availableConnections, IDialogService dialogService, Action onChanged)
|
||||
{
|
||||
Name = name ?? throw new ArgumentNullException(nameof(name));
|
||||
_model = model ?? throw new ArgumentNullException(nameof(model));
|
||||
AvailableConnections = availableConnections ?? [];
|
||||
_dialogService = dialogService ?? throw new ArgumentNullException(nameof(dialogService));
|
||||
_onChanged = onChanged ?? throw new ArgumentNullException(nameof(onChanged));
|
||||
|
||||
// Initialize collections
|
||||
@@ -44,8 +47,11 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
AddTransformerCommand = new RelayCommand(AddTransformer);
|
||||
AddPostScriptCommand = new RelayCommand(AddPostScript);
|
||||
RemoveStepCommand = new RelayCommand<PipelineStepViewModelBase>(RemoveStep);
|
||||
DeleteSelectedStepCommand = new AsyncRelayCommand(DeleteSelectedStepAsync, () => CanDeleteSelectedStep);
|
||||
MoveStepUpCommand = new RelayCommand<PipelineStepViewModelBase>(MoveStepUp, CanMoveStepUp);
|
||||
MoveStepDownCommand = new RelayCommand<PipelineStepViewModelBase>(MoveStepDown, CanMoveStepDown);
|
||||
MoveSelectedStepUpCommand = new RelayCommand(MoveSelectedStepUp, () => CanMoveSelectedStepUp);
|
||||
MoveSelectedStepDownCommand = new RelayCommand(MoveSelectedStepDown, () => CanMoveSelectedStepDown);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -122,11 +128,35 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
_selectedStep.IsSelected = true;
|
||||
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(CanDeleteSelectedStep));
|
||||
OnPropertyChanged(nameof(CanMoveSelectedStepUp));
|
||||
OnPropertyChanged(nameof(CanMoveSelectedStepDown));
|
||||
(DeleteSelectedStepCommand as AsyncRelayCommand)?.RaiseCanExecuteChanged();
|
||||
(MoveSelectedStepUpCommand as RelayCommand)?.RaiseCanExecuteChanged();
|
||||
(MoveSelectedStepDownCommand as RelayCommand)?.RaiseCanExecuteChanged();
|
||||
UpdateSelectedStepEditor();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the selected step can be deleted.
|
||||
/// Source and Destination steps cannot be deleted.
|
||||
/// </summary>
|
||||
public bool CanDeleteSelectedStep => _selectedStep is PreScriptStepViewModel
|
||||
or TransformerStepViewModelBase
|
||||
or PostScriptStepViewModel;
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the selected step can be moved up.
|
||||
/// </summary>
|
||||
public bool CanMoveSelectedStepUp => CanMoveStepUp(_selectedStep);
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the selected step can be moved down.
|
||||
/// </summary>
|
||||
public bool CanMoveSelectedStepDown => CanMoveStepDown(_selectedStep);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the editor view model for the currently selected step.
|
||||
/// </summary>
|
||||
@@ -156,8 +186,11 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
public ICommand AddTransformerCommand { get; }
|
||||
public ICommand AddPostScriptCommand { get; }
|
||||
public ICommand RemoveStepCommand { get; }
|
||||
public ICommand DeleteSelectedStepCommand { get; }
|
||||
public ICommand MoveStepUpCommand { get; }
|
||||
public ICommand MoveStepDownCommand { get; }
|
||||
public ICommand MoveSelectedStepUpCommand { get; }
|
||||
public ICommand MoveSelectedStepDownCommand { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the list of available transformer types for the add dialog.
|
||||
@@ -191,7 +224,7 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
}
|
||||
|
||||
// Source
|
||||
Source = new SourceStepViewModel(_model.Source, () =>
|
||||
Source = new SourceStepViewModel(_model.Source, AvailableConnections, () =>
|
||||
{
|
||||
_onChanged();
|
||||
});
|
||||
@@ -411,6 +444,28 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
OnPropertyChanged(nameof(AllSteps));
|
||||
}
|
||||
|
||||
private void MoveSelectedStepUp()
|
||||
{
|
||||
if (_selectedStep == null) return;
|
||||
MoveStepUp(_selectedStep);
|
||||
RaiseMoveCanExecuteChanged();
|
||||
}
|
||||
|
||||
private void MoveSelectedStepDown()
|
||||
{
|
||||
if (_selectedStep == null) return;
|
||||
MoveStepDown(_selectedStep);
|
||||
RaiseMoveCanExecuteChanged();
|
||||
}
|
||||
|
||||
private void RaiseMoveCanExecuteChanged()
|
||||
{
|
||||
OnPropertyChanged(nameof(CanMoveSelectedStepUp));
|
||||
OnPropertyChanged(nameof(CanMoveSelectedStepDown));
|
||||
(MoveSelectedStepUpCommand as RelayCommand)?.RaiseCanExecuteChanged();
|
||||
(MoveSelectedStepDownCommand as RelayCommand)?.RaiseCanExecuteChanged();
|
||||
}
|
||||
|
||||
private static void MoveInCollection<T>(ObservableCollection<T> collection, T item, int offset)
|
||||
{
|
||||
var index = collection.IndexOf(item);
|
||||
@@ -440,4 +495,31 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
? PostScripts.Select(s => s.Script).ToArray()
|
||||
: null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes the currently selected step after user confirmation.
|
||||
/// Source and Destination steps cannot be deleted.
|
||||
/// </summary>
|
||||
private async Task DeleteSelectedStepAsync()
|
||||
{
|
||||
if (_selectedStep == null || !CanDeleteSelectedStep)
|
||||
return;
|
||||
|
||||
var stepTypeName = _selectedStep switch
|
||||
{
|
||||
PreScriptStepViewModel => "Pre-Script",
|
||||
TransformerStepViewModelBase t => $"Transformer ({t.TransformerType})",
|
||||
PostScriptStepViewModel => "Post-Script",
|
||||
_ => "Step"
|
||||
};
|
||||
|
||||
var confirmed = await _dialogService.ShowConfirmationAsync(
|
||||
"Delete Step",
|
||||
$"Are you sure you want to delete this {stepTypeName}?\n\nThis action cannot be undone.");
|
||||
|
||||
if (!confirmed)
|
||||
return;
|
||||
|
||||
RemoveStep(_selectedStep);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -573,9 +573,9 @@ public class MainWindowViewModel : ViewModelBase
|
||||
MarkAsChanged,
|
||||
_dialogService,
|
||||
_connectionTestService),
|
||||
_ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null
|
||||
_ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null && _dialogService != null
|
||||
=> _pipelines.Pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline)
|
||||
? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), MarkAsChanged)
|
||||
? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), _dialogService, MarkAsChanged)
|
||||
: null,
|
||||
_ => null
|
||||
};
|
||||
|
||||
+7
-1
@@ -20,9 +20,10 @@ public class SourceStepViewModel : PipelineStepViewModelBase
|
||||
{
|
||||
private readonly PipelineSource _model;
|
||||
|
||||
public SourceStepViewModel(PipelineSource model, Action onChanged) : base(onChanged)
|
||||
public SourceStepViewModel(PipelineSource model, IReadOnlyList<string> availableConnections, Action onChanged) : base(onChanged)
|
||||
{
|
||||
_model = model ?? throw new ArgumentNullException(nameof(model));
|
||||
AvailableConnections = availableConnections ?? [];
|
||||
|
||||
// Initialize parameters collection
|
||||
Parameters = new ObservableCollection<ParameterViewModel>(
|
||||
@@ -36,6 +37,11 @@ public class SourceStepViewModel : PipelineStepViewModelBase
|
||||
AddParameterCommand = new RelayCommand(AddParameter);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the list of available connection string names from configuration.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> AvailableConnections { get; }
|
||||
|
||||
public override PipelineStepType StepType => PipelineStepType.Source;
|
||||
public override string DisplayName => "Source";
|
||||
public override string Icon => IsFileSource ? "" : ""; // mdi-file vs mdi-database
|
||||
|
||||
@@ -39,12 +39,13 @@
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding Connection}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="36"
|
||||
FontFamily="JetBrains Mono"
|
||||
Watermark="jde"/>
|
||||
<TextBlock Text="Connection string name (e.g., jde, cms, giw)"
|
||||
<ComboBox ItemsSource="{Binding AvailableConnections}"
|
||||
SelectedItem="{Binding Connection}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="36"
|
||||
HorizontalAlignment="Stretch"
|
||||
PlaceholderText="Select connection..."/>
|
||||
<TextBlock Text="Connection string name from Settings > ConnectionStrings"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
|
||||
+65
-34
@@ -279,40 +279,71 @@
|
||||
</Border>
|
||||
</StackPanel>
|
||||
|
||||
<!-- DataGrid -->
|
||||
<DataGrid ItemsSource="{Binding Connections}"
|
||||
SelectedItem="{Binding SelectedConnection}"
|
||||
SelectionMode="Single"
|
||||
Height="200"
|
||||
Background="#0D0F12"
|
||||
RowBackground="#0D0F12"
|
||||
BorderBrush="#2D3540"
|
||||
BorderThickness="1"
|
||||
GridLinesVisibility="Horizontal"
|
||||
HorizontalGridLinesBrush="#2D3540"
|
||||
HeadersVisibility="Column"
|
||||
AutoGenerateColumns="False"
|
||||
CanUserReorderColumns="False"
|
||||
CanUserResizeColumns="True"
|
||||
CanUserSortColumns="True">
|
||||
<DataGrid.Columns>
|
||||
<DataGridTextColumn Header="Name"
|
||||
Binding="{Binding Name}"
|
||||
Width="*"
|
||||
IsReadOnly="True"
|
||||
Foreground="#E6EDF5"/>
|
||||
<DataGridTextColumn Header="Provider"
|
||||
Binding="{Binding ProviderDisplay}"
|
||||
Width="100"
|
||||
IsReadOnly="True"
|
||||
Foreground="#9BA8B8"/>
|
||||
<DataGridTextColumn Header="Server"
|
||||
Binding="{Binding ServerDisplay}"
|
||||
Width="150"
|
||||
IsReadOnly="True"
|
||||
Foreground="#9BA8B8"/>
|
||||
</DataGrid.Columns>
|
||||
</DataGrid>
|
||||
<!-- Connection List -->
|
||||
<Border BorderBrush="#2D3540" BorderThickness="1" CornerRadius="4">
|
||||
<StackPanel>
|
||||
<!-- Header Row -->
|
||||
<Grid Background="#151920" Height="32">
|
||||
<Grid.ColumnDefinitions>
|
||||
<ColumnDefinition Width="*"/>
|
||||
<ColumnDefinition Width="100"/>
|
||||
<ColumnDefinition Width="150"/>
|
||||
</Grid.ColumnDefinitions>
|
||||
<TextBlock Grid.Column="0" Text="Name" Foreground="#9BA8B8"
|
||||
FontSize="12" FontWeight="Medium"
|
||||
VerticalAlignment="Center" Margin="12,0"/>
|
||||
<TextBlock Grid.Column="1" Text="Provider" Foreground="#9BA8B8"
|
||||
FontSize="12" FontWeight="Medium"
|
||||
VerticalAlignment="Center" Margin="8,0"/>
|
||||
<TextBlock Grid.Column="2" Text="Server" Foreground="#9BA8B8"
|
||||
FontSize="12" FontWeight="Medium"
|
||||
VerticalAlignment="Center" Margin="8,0"/>
|
||||
</Grid>
|
||||
<!-- List Items -->
|
||||
<ListBox ItemsSource="{Binding Connections}"
|
||||
SelectedItem="{Binding SelectedConnection}"
|
||||
SelectionMode="Single"
|
||||
Background="#0D0F12"
|
||||
MaxHeight="180"
|
||||
MinHeight="100">
|
||||
<ListBox.Styles>
|
||||
<Style Selector="ListBoxItem">
|
||||
<Setter Property="Padding" Value="0"/>
|
||||
<Setter Property="Margin" Value="0"/>
|
||||
<Setter Property="Background" Value="Transparent"/>
|
||||
</Style>
|
||||
<Style Selector="ListBoxItem:selected /template/ ContentPresenter">
|
||||
<Setter Property="Background" Value="#2563EB"/>
|
||||
</Style>
|
||||
<Style Selector="ListBoxItem:pointerover /template/ ContentPresenter">
|
||||
<Setter Property="Background" Value="#1E3A5F"/>
|
||||
</Style>
|
||||
</ListBox.Styles>
|
||||
<ListBox.ItemTemplate>
|
||||
<DataTemplate>
|
||||
<Grid Height="36" Background="Transparent">
|
||||
<Grid.ColumnDefinitions>
|
||||
<ColumnDefinition Width="*"/>
|
||||
<ColumnDefinition Width="100"/>
|
||||
<ColumnDefinition Width="150"/>
|
||||
</Grid.ColumnDefinitions>
|
||||
<TextBlock Grid.Column="0" Text="{Binding Name}"
|
||||
Foreground="#E6EDF5" FontSize="13"
|
||||
VerticalAlignment="Center" Margin="12,0"/>
|
||||
<TextBlock Grid.Column="1" Text="{Binding ProviderDisplay}"
|
||||
Foreground="#9BA8B8" FontSize="13"
|
||||
VerticalAlignment="Center" Margin="8,0"/>
|
||||
<TextBlock Grid.Column="2" Text="{Binding ServerDisplay}"
|
||||
Foreground="#9BA8B8" FontSize="13"
|
||||
VerticalAlignment="Center" Margin="8,0"/>
|
||||
<Border Grid.ColumnSpan="3" BorderBrush="#2D3540"
|
||||
BorderThickness="0,0,0,1" VerticalAlignment="Bottom"/>
|
||||
</Grid>
|
||||
</DataTemplate>
|
||||
</ListBox.ItemTemplate>
|
||||
</ListBox>
|
||||
</StackPanel>
|
||||
</Border>
|
||||
|
||||
<!-- Toolbar -->
|
||||
<StackPanel Orientation="Horizontal" Spacing="8">
|
||||
|
||||
@@ -188,12 +188,49 @@
|
||||
<Border Grid.Column="4" Background="#0D0F12" Padding="16">
|
||||
<ScrollViewer VerticalScrollBarVisibility="Auto">
|
||||
<StackPanel Spacing="16">
|
||||
<!-- Properties Header -->
|
||||
<StackPanel>
|
||||
<TextBlock Text="PROPERTIES"
|
||||
Foreground="#5C6A7A" FontSize="11" FontWeight="Medium"/>
|
||||
<Border Height="1" Background="#2D3540" Margin="0,8,0,0"/>
|
||||
</StackPanel>
|
||||
<!-- Properties Header with Action Buttons -->
|
||||
<Grid>
|
||||
<Grid.ColumnDefinitions>
|
||||
<ColumnDefinition Width="*"/>
|
||||
<ColumnDefinition Width="Auto"/>
|
||||
</Grid.ColumnDefinitions>
|
||||
<TextBlock Grid.Column="0" Text="PROPERTIES"
|
||||
Foreground="#5C6A7A" FontSize="11" FontWeight="Medium"
|
||||
VerticalAlignment="Center"/>
|
||||
<StackPanel Grid.Column="1" Orientation="Horizontal" Spacing="4">
|
||||
<!-- Move Up Button -->
|
||||
<Button Command="{Binding MoveSelectedStepUpCommand}"
|
||||
Background="#1F2937" Foreground="#9BA8B8"
|
||||
BorderBrush="#3D4550" BorderThickness="1"
|
||||
Height="28" Width="28" Padding="0"
|
||||
ToolTip.Tip="Move step up"
|
||||
IsVisible="{Binding CanDeleteSelectedStep}">
|
||||
<TextBlock Text="▲" FontSize="10" HorizontalAlignment="Center"/>
|
||||
</Button>
|
||||
<!-- Move Down Button -->
|
||||
<Button Command="{Binding MoveSelectedStepDownCommand}"
|
||||
Background="#1F2937" Foreground="#9BA8B8"
|
||||
BorderBrush="#3D4550" BorderThickness="1"
|
||||
Height="28" Width="28" Padding="0"
|
||||
ToolTip.Tip="Move step down"
|
||||
IsVisible="{Binding CanDeleteSelectedStep}">
|
||||
<TextBlock Text="▼" FontSize="10" HorizontalAlignment="Center"/>
|
||||
</Button>
|
||||
<!-- Delete Button -->
|
||||
<Button Command="{Binding DeleteSelectedStepCommand}"
|
||||
Background="#3D1F1F" Foreground="#FF6B6B"
|
||||
BorderBrush="#5C2D2D" BorderThickness="1"
|
||||
Height="28" Padding="12,0"
|
||||
ToolTip.Tip="Delete selected step"
|
||||
IsVisible="{Binding CanDeleteSelectedStep}">
|
||||
<StackPanel Orientation="Horizontal" Spacing="6">
|
||||
<TextBlock Text="🗑" FontSize="12"/>
|
||||
<TextBlock Text="Delete" FontSize="11"/>
|
||||
</StackPanel>
|
||||
</Button>
|
||||
</StackPanel>
|
||||
</Grid>
|
||||
<Border Height="1" Background="#2D3540" Margin="0,8,0,0"/>
|
||||
|
||||
<!-- Step Editor Content (changes based on selection) -->
|
||||
<!-- Shows placeholder text when nothing selected, otherwise uses DataTemplates from MainWindow -->
|
||||
|
||||
@@ -36,6 +36,9 @@
|
||||
<DataTemplate DataType="{x:Type forms:ExcelExportFormViewModel}">
|
||||
<views:ExcelExportFormView/>
|
||||
</DataTemplate>
|
||||
<DataTemplate DataType="{x:Type forms:ConnectionStringsFormViewModel}">
|
||||
<views:ConnectionStringsFormView/>
|
||||
</DataTemplate>
|
||||
|
||||
<!-- Pipeline Editor (replaces PipelineFormViewModel) -->
|
||||
<DataTemplate DataType="{x:Type forms:PipelineEditorViewModel}">
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
using System.Security.Claims;
|
||||
using JdeScoping.Api.Contracts.ManualSync;
|
||||
using JdeScoping.Api.Controllers;
|
||||
using JdeScoping.DataAccess.Services;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using JdeScoping.Domain.Models;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace JdeScoping.Api.Tests.Controllers;
|
||||
|
||||
public class ManualSyncControllerTests
|
||||
{
|
||||
private readonly IManualSyncRequestService _manualSyncRequestService;
|
||||
private readonly IPipelineRegistry _pipelineRegistry;
|
||||
private readonly ManualSyncController _controller;
|
||||
|
||||
public ManualSyncControllerTests()
|
||||
{
|
||||
_manualSyncRequestService = Substitute.For<IManualSyncRequestService>();
|
||||
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
|
||||
_controller = new ManualSyncController(_manualSyncRequestService, _pipelineRegistry);
|
||||
SetupAuthenticatedUser("testuser");
|
||||
}
|
||||
|
||||
#region GetRequests Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetRequests_ReturnsOkWithRequests()
|
||||
{
|
||||
// Arrange
|
||||
var requests = new List<ManualSyncRequest>
|
||||
{
|
||||
CreateRequest(1, "Pipeline1", "mass", "user1", DateTime.UtcNow.AddHours(-2)),
|
||||
CreateRequest(2, "Pipeline2", "daily", "user2", DateTime.UtcNow.AddHours(-1))
|
||||
};
|
||||
_manualSyncRequestService.GetRequestsAsync(false, Arg.Any<CancellationToken>())
|
||||
.Returns(requests);
|
||||
|
||||
// Act
|
||||
var result = await _controller.GetRequests(false, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<ManualSyncRequestViewModel>>()!;
|
||||
viewModels.Count.ShouldBe(2);
|
||||
viewModels[0].Id.ShouldBe(1);
|
||||
viewModels[0].PipelineName.ShouldBe("Pipeline1");
|
||||
viewModels[1].Id.ShouldBe(2);
|
||||
viewModels[1].PipelineName.ShouldBe("Pipeline2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetRequests_WithPendingOnlyTrue_PassesPendingOnlyToService()
|
||||
{
|
||||
// Arrange
|
||||
_manualSyncRequestService.GetRequestsAsync(true, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<ManualSyncRequest>());
|
||||
|
||||
// Act
|
||||
await _controller.GetRequests(true, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
await _manualSyncRequestService.Received(1)
|
||||
.GetRequestsAsync(true, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetRequests_MapsRowVersionToBase64()
|
||||
{
|
||||
// Arrange
|
||||
var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
|
||||
var request = CreateRequest(1, "Pipeline1", "mass", "user1", DateTime.UtcNow, rowVersion: rowVersion);
|
||||
_manualSyncRequestService.GetRequestsAsync(false, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<ManualSyncRequest> { request });
|
||||
|
||||
// Act
|
||||
var result = await _controller.GetRequests(false, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<ManualSyncRequestViewModel>>()!;
|
||||
viewModels[0].RowVersionBase64.ShouldBe(Convert.ToBase64String(rowVersion));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region GetPipelines Tests
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_ReturnsOkWithPipelines()
|
||||
{
|
||||
// Arrange
|
||||
var pipelines = new List<EtlPipelineConfig>
|
||||
{
|
||||
CreatePipeline("WorkOrders", massSyncInterval: 1440, dailySyncInterval: 60, hourlySyncInterval: 15),
|
||||
CreatePipeline("Items", massSyncInterval: 1440, dailySyncInterval: 60)
|
||||
};
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<PipelineInfoViewModel>>()!;
|
||||
viewModels.Count.ShouldBe(2);
|
||||
viewModels[0].Name.ShouldBe("WorkOrders");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("daily");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("hourly");
|
||||
viewModels[1].Name.ShouldBe("Items");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_WhenEmpty_ReturnsEmptyList()
|
||||
{
|
||||
// Arrange
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(new List<EtlPipelineConfig>());
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<PipelineInfoViewModel>>()!;
|
||||
viewModels.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region CreateRequest Tests
|
||||
|
||||
[Fact]
|
||||
public async Task CreateRequest_WithValidInput_ReturnsCreated()
|
||||
{
|
||||
// Arrange
|
||||
var dto = new CreateManualSyncRequestDto
|
||||
{
|
||||
PipelineName = "WorkOrders",
|
||||
SyncType = "mass"
|
||||
};
|
||||
var createdRequest = CreateRequest(42, "WorkOrders", "mass", "testuser", DateTime.UtcNow);
|
||||
|
||||
_pipelineRegistry.IsValidPipelineAndSyncType("WorkOrders", "mass").Returns(true);
|
||||
_manualSyncRequestService.CreateRequestAsync("WorkOrders", "mass", "testuser", Arg.Any<CancellationToken>())
|
||||
.Returns(createdRequest);
|
||||
|
||||
// Act
|
||||
var result = await _controller.CreateRequest(dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<CreatedAtActionResult>();
|
||||
var createdResult = (CreatedAtActionResult)result.Result!;
|
||||
var viewModel = createdResult.Value.ShouldBeOfType<ManualSyncRequestViewModel>();
|
||||
viewModel.Id.ShouldBe(42);
|
||||
viewModel.PipelineName.ShouldBe("WorkOrders");
|
||||
viewModel.SyncType.ShouldBe("mass");
|
||||
viewModel.RequestedBy.ShouldBe("testuser");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CreateRequest_WithInvalidPipelineOrSyncType_ReturnsBadRequest()
|
||||
{
|
||||
// Arrange
|
||||
var dto = new CreateManualSyncRequestDto
|
||||
{
|
||||
PipelineName = "InvalidPipeline",
|
||||
SyncType = "invalid"
|
||||
};
|
||||
_pipelineRegistry.IsValidPipelineAndSyncType("InvalidPipeline", "invalid").Returns(false);
|
||||
|
||||
// Act
|
||||
var result = await _controller.CreateRequest(dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<BadRequestObjectResult>();
|
||||
var badRequestResult = (BadRequestObjectResult)result.Result!;
|
||||
badRequestResult.Value.ShouldBeOfType<string>();
|
||||
((string)badRequestResult.Value!).ShouldContain("Invalid pipeline/sync type combination");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CreateRequest_WhenUnauthenticated_ReturnsUnauthorized()
|
||||
{
|
||||
// Arrange
|
||||
SetupUnauthenticatedUser();
|
||||
var dto = new CreateManualSyncRequestDto
|
||||
{
|
||||
PipelineName = "WorkOrders",
|
||||
SyncType = "mass"
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await _controller.CreateRequest(dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<UnauthorizedResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CreateRequest_PassesCorrectUsernameToService()
|
||||
{
|
||||
// Arrange
|
||||
var dto = new CreateManualSyncRequestDto
|
||||
{
|
||||
PipelineName = "WorkOrders",
|
||||
SyncType = "daily"
|
||||
};
|
||||
var createdRequest = CreateRequest(1, "WorkOrders", "daily", "testuser", DateTime.UtcNow);
|
||||
|
||||
_pipelineRegistry.IsValidPipelineAndSyncType("WorkOrders", "daily").Returns(true);
|
||||
_manualSyncRequestService.CreateRequestAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
|
||||
.Returns(createdRequest);
|
||||
|
||||
// Act
|
||||
await _controller.CreateRequest(dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
await _manualSyncRequestService.Received(1)
|
||||
.CreateRequestAsync("WorkOrders", "daily", "testuser", Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region CancelRequest Tests
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_WhenSuccessful_ReturnsOk()
|
||||
{
|
||||
// Arrange
|
||||
var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = Convert.ToBase64String(rowVersion)
|
||||
};
|
||||
_manualSyncRequestService.CancelRequestAsync(1, "testuser", Arg.Any<byte[]>(), Arg.Any<CancellationToken>())
|
||||
.Returns(true);
|
||||
|
||||
// Act
|
||||
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeOfType<OkObjectResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_WhenConcurrencyFails_ReturnsConflict()
|
||||
{
|
||||
// Arrange
|
||||
var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = Convert.ToBase64String(rowVersion)
|
||||
};
|
||||
_manualSyncRequestService.CancelRequestAsync(1, "testuser", Arg.Any<byte[]>(), Arg.Any<CancellationToken>())
|
||||
.Returns(false);
|
||||
|
||||
// Act
|
||||
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeOfType<ConflictObjectResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_WithInvalidBase64_ReturnsBadRequest()
|
||||
{
|
||||
// Arrange
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = "not-valid-base64!!!"
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeOfType<BadRequestObjectResult>();
|
||||
var badRequest = (BadRequestObjectResult)result;
|
||||
badRequest.Value.ShouldBeOfType<string>();
|
||||
((string)badRequest.Value!).ShouldContain("Invalid RowVersionBase64 format");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_WhenUnauthenticated_ReturnsUnauthorized()
|
||||
{
|
||||
// Arrange
|
||||
SetupUnauthenticatedUser();
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = Convert.ToBase64String(new byte[] { 1, 2, 3 })
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeOfType<UnauthorizedResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_PassesCorrectParametersToService()
|
||||
{
|
||||
// Arrange
|
||||
var rowVersion = new byte[] { 10, 20, 30, 40 };
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = Convert.ToBase64String(rowVersion)
|
||||
};
|
||||
_manualSyncRequestService.CancelRequestAsync(Arg.Any<int>(), Arg.Any<string>(), Arg.Any<byte[]>(), Arg.Any<CancellationToken>())
|
||||
.Returns(true);
|
||||
|
||||
// Act
|
||||
await _controller.CancelRequest(99, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
await _manualSyncRequestService.Received(1)
|
||||
.CancelRequestAsync(
|
||||
99,
|
||||
"testuser",
|
||||
Arg.Is<byte[]>(b => b.SequenceEqual(rowVersion)),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private void SetupAuthenticatedUser(string username)
|
||||
{
|
||||
var claims = new List<Claim>
|
||||
{
|
||||
new(ClaimTypes.Name, username),
|
||||
new("dn", $"CN={username},DC=example,DC=com")
|
||||
};
|
||||
var identity = new ClaimsIdentity(claims, "Test");
|
||||
var principal = new ClaimsPrincipal(identity);
|
||||
|
||||
var httpContext = new DefaultHttpContext { User = principal };
|
||||
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
|
||||
}
|
||||
|
||||
private void SetupUnauthenticatedUser()
|
||||
{
|
||||
var identity = new ClaimsIdentity(); // No claims, not authenticated
|
||||
var principal = new ClaimsPrincipal(identity);
|
||||
|
||||
var httpContext = new DefaultHttpContext { User = principal };
|
||||
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
|
||||
}
|
||||
|
||||
private static ManualSyncRequest CreateRequest(
|
||||
int id,
|
||||
string pipelineName,
|
||||
string syncType,
|
||||
string requestedBy,
|
||||
DateTime requestDT,
|
||||
DateTime? completedDT = null,
|
||||
DateTime? cancelDT = null,
|
||||
string? cancelledBy = null,
|
||||
byte[]? rowVersion = null)
|
||||
{
|
||||
return new ManualSyncRequest
|
||||
{
|
||||
Id = id,
|
||||
PipelineName = pipelineName,
|
||||
SyncType = syncType,
|
||||
RequestedBy = requestedBy,
|
||||
RequestDT = requestDT,
|
||||
CompletedDT = completedDT,
|
||||
CancelDT = cancelDT,
|
||||
CancelledBy = cancelledBy,
|
||||
RowVersion = rowVersion ?? new byte[] { 0, 0, 0, 0, 0, 0, 0, 1 }
|
||||
};
|
||||
}
|
||||
|
||||
private static EtlPipelineConfig CreatePipeline(
|
||||
string name,
|
||||
int? massSyncInterval = null,
|
||||
int? dailySyncInterval = null,
|
||||
int? hourlySyncInterval = null,
|
||||
bool isEnabled = true)
|
||||
{
|
||||
return new EtlPipelineConfig
|
||||
{
|
||||
Name = name,
|
||||
IsEnabled = isEnabled,
|
||||
MassSyncIntervalMinutes = massSyncInterval,
|
||||
DailySyncIntervalMinutes = dailySyncInterval,
|
||||
HourlySyncIntervalMinutes = hourlySyncInterval
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
using System.Security.Claims;
|
||||
using JdeScoping.Api.Controllers;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NSubstitute;
|
||||
using NSubstitute.ExceptionExtensions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace JdeScoping.Api.Tests.Controllers;
|
||||
|
||||
public class PipelineControllerTests
|
||||
{
|
||||
private readonly IPipelineRegistry _pipelineRegistry;
|
||||
private readonly ILogger<PipelineController> _logger;
|
||||
private readonly PipelineController _controller;
|
||||
|
||||
public PipelineControllerTests()
|
||||
{
|
||||
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
|
||||
_logger = Substitute.For<ILogger<PipelineController>>();
|
||||
_controller = new PipelineController(_pipelineRegistry, _logger);
|
||||
SetupAuthenticatedUser("testuser", isAdmin: false);
|
||||
}
|
||||
|
||||
#region GetPipelines Tests
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_ReturnsEnabledPipelines()
|
||||
{
|
||||
// Arrange
|
||||
var pipelines = new List<EtlPipelineConfig>
|
||||
{
|
||||
CreatePipeline("WorkOrders", massSyncInterval: 1440, dailySyncInterval: 60, hourlySyncInterval: 15),
|
||||
CreatePipeline("Items", massSyncInterval: 1440, dailySyncInterval: 60)
|
||||
};
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
|
||||
|
||||
viewModels.Count.ShouldBe(2);
|
||||
viewModels[0].Name.ShouldBe("WorkOrders");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("daily");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("hourly");
|
||||
viewModels[1].Name.ShouldBe("Items");
|
||||
viewModels[1].SupportedSyncTypes.ShouldNotContain("hourly");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_WhenEmpty_ReturnsEmptyList()
|
||||
{
|
||||
// Arrange
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(new List<EtlPipelineConfig>());
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
|
||||
viewModels.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_MapsOnlySupportedSyncTypes()
|
||||
{
|
||||
// Arrange - pipeline with only mass sync
|
||||
var pipelines = new List<EtlPipelineConfig>
|
||||
{
|
||||
CreatePipeline("MassOnly", massSyncInterval: 1440)
|
||||
};
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
|
||||
|
||||
viewModels[0].SupportedSyncTypes.Count.ShouldBe(1);
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region GetStatus Tests
|
||||
|
||||
[Fact]
|
||||
public void GetStatus_ReturnsRegistryMetadata()
|
||||
{
|
||||
// Arrange
|
||||
var allPipelines = new List<EtlPipelineConfig>
|
||||
{
|
||||
CreatePipeline("Pipeline1", isEnabled: true),
|
||||
CreatePipeline("Pipeline2", isEnabled: true),
|
||||
CreatePipeline("Pipeline3", isEnabled: false)
|
||||
};
|
||||
var enabledPipelines = allPipelines.Where(p => p.IsEnabled).ToList();
|
||||
|
||||
_pipelineRegistry.GetAllPipelines().Returns(allPipelines);
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(enabledPipelines);
|
||||
_pipelineRegistry.Version.Returns(5);
|
||||
_pipelineRegistry.LastLoadedAt.Returns(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc));
|
||||
|
||||
// Act
|
||||
var result = _controller.GetStatus();
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var status = okResult.Value.ShouldBeOfType<PipelineRegistryStatusViewModel>();
|
||||
|
||||
status.Version.ShouldBe(5);
|
||||
status.LastLoadedAt.ShouldBe(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc));
|
||||
status.TotalPipelines.ShouldBe(3);
|
||||
status.EnabledPipelines.ShouldBe(2);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region ReloadPipelines Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadPipelines_CallsRegistry()
|
||||
{
|
||||
// Arrange
|
||||
SetupAuthenticatedUser("admin", isAdmin: true);
|
||||
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(CreateSuccessfulReloadResult(5, 0));
|
||||
|
||||
// Act
|
||||
await _controller.ReloadPipelines(CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
await _pipelineRegistry.Received(1).ReloadAsync(Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadPipelines_ReturnsResult()
|
||||
{
|
||||
// Arrange
|
||||
SetupAuthenticatedUser("admin", isAdmin: true);
|
||||
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(CreateSuccessfulReloadResult(pipelinesLoaded: 10, pipelinesSkipped: 2, previousVersion: 3, newVersion: 4));
|
||||
|
||||
// Act
|
||||
var result = await _controller.ReloadPipelines(CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var reloadResult = okResult.Value.ShouldBeOfType<PipelineReloadResultViewModel>();
|
||||
|
||||
reloadResult.Success.ShouldBeTrue();
|
||||
reloadResult.PipelinesLoaded.ShouldBe(10);
|
||||
reloadResult.PipelinesSkipped.ShouldBe(2);
|
||||
reloadResult.PreviousVersion.ShouldBe(3);
|
||||
reloadResult.NewVersion.ShouldBe(4);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadPipelines_WithErrors_ReturnsErrorDetails()
|
||||
{
|
||||
// Arrange
|
||||
SetupAuthenticatedUser("admin", isAdmin: true);
|
||||
var errors = new List<PipelineLoadError>
|
||||
{
|
||||
new PipelineLoadError
|
||||
{
|
||||
FileName = "pipeline.Bad.json",
|
||||
PipelineName = "Bad",
|
||||
ErrorType = "Validation",
|
||||
Messages = new List<string> { "Missing source", "Missing destination" }
|
||||
}
|
||||
};
|
||||
|
||||
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new PipelineReloadResult
|
||||
{
|
||||
Success = false,
|
||||
PipelinesLoaded = 5,
|
||||
PipelinesSkipped = 1,
|
||||
PreviousVersion = 1,
|
||||
NewVersion = 1,
|
||||
Errors = errors
|
||||
});
|
||||
|
||||
// Act
|
||||
var result = await _controller.ReloadPipelines(CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var reloadResult = okResult.Value.ShouldBeOfType<PipelineReloadResultViewModel>();
|
||||
|
||||
reloadResult.Success.ShouldBeFalse();
|
||||
reloadResult.Errors.Count.ShouldBe(1);
|
||||
reloadResult.Errors[0].FileName.ShouldBe("pipeline.Bad.json");
|
||||
reloadResult.Errors[0].ErrorType.ShouldBe("Validation");
|
||||
reloadResult.Errors[0].Messages.ShouldContain("Missing source");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadPipelines_WhenExceptionThrown_Returns500()
|
||||
{
|
||||
// Arrange
|
||||
SetupAuthenticatedUser("admin", isAdmin: true);
|
||||
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
|
||||
.Throws(new Exception("Unexpected error"));
|
||||
|
||||
// Act
|
||||
var result = await _controller.ReloadPipelines(CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<ObjectResult>();
|
||||
var objectResult = (ObjectResult)result.Result!;
|
||||
objectResult.StatusCode.ShouldBe(StatusCodes.Status500InternalServerError);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private void SetupAuthenticatedUser(string username, bool isAdmin = false)
|
||||
{
|
||||
var claims = new List<Claim>
|
||||
{
|
||||
new(ClaimTypes.Name, username)
|
||||
};
|
||||
|
||||
if (isAdmin)
|
||||
{
|
||||
claims.Add(new Claim(ClaimTypes.Role, "Admin"));
|
||||
}
|
||||
|
||||
var identity = new ClaimsIdentity(claims, "Test");
|
||||
var principal = new ClaimsPrincipal(identity);
|
||||
|
||||
var httpContext = new DefaultHttpContext { User = principal };
|
||||
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
|
||||
}
|
||||
|
||||
private static EtlPipelineConfig CreatePipeline(
|
||||
string name,
|
||||
bool isEnabled = true,
|
||||
int? massSyncInterval = null,
|
||||
int? dailySyncInterval = null,
|
||||
int? hourlySyncInterval = null)
|
||||
{
|
||||
return new EtlPipelineConfig
|
||||
{
|
||||
Name = name,
|
||||
IsEnabled = isEnabled,
|
||||
MassSyncIntervalMinutes = massSyncInterval,
|
||||
DailySyncIntervalMinutes = dailySyncInterval,
|
||||
HourlySyncIntervalMinutes = hourlySyncInterval
|
||||
};
|
||||
}
|
||||
|
||||
private static PipelineReloadResult CreateSuccessfulReloadResult(
|
||||
int pipelinesLoaded,
|
||||
int pipelinesSkipped,
|
||||
int previousVersion = 1,
|
||||
int newVersion = 2)
|
||||
{
|
||||
return new PipelineReloadResult
|
||||
{
|
||||
Success = true,
|
||||
PipelinesLoaded = pipelinesLoaded,
|
||||
PipelinesSkipped = pipelinesSkipped,
|
||||
PreviousVersion = previousVersion,
|
||||
NewVersion = newVersion,
|
||||
Errors = new List<PipelineLoadError>()
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
using System.Text.Json;
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Tests.Models;
|
||||
|
||||
public class ConnectionStringsSectionConverterTests
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Deserialize_StandardDictionaryFormat_ParsesAllConnections()
|
||||
{
|
||||
// Arrange
|
||||
var json = """
|
||||
{
|
||||
"LotFinder": "Server=localhost,1434;Database=ScopingTool;User Id=sa;Password=test",
|
||||
"JDE": "Data Source=jde-server:1521/JDEPROD;User Id=jdeuser;Password=jdepass",
|
||||
"CMS": "Data Source=cms-server:1521/CMSPROD;User Id=cmsuser;Password=cmspass"
|
||||
}
|
||||
""";
|
||||
|
||||
// Act
|
||||
var section = JsonSerializer.Deserialize<ConnectionStringsSection>(json, JsonOptions);
|
||||
|
||||
// Assert
|
||||
section.ShouldNotBeNull();
|
||||
section.Entries.Count.ShouldBe(3);
|
||||
|
||||
var lotFinder = section.Entries.First(e => e.Name == "LotFinder");
|
||||
lotFinder.Provider.ShouldBe(ConnectionProvider.SqlServer);
|
||||
lotFinder.Server.ShouldBe("localhost,1434");
|
||||
lotFinder.Database.ShouldBe("ScopingTool");
|
||||
lotFinder.UserId.ShouldBe("sa");
|
||||
|
||||
var jde = section.Entries.First(e => e.Name == "JDE");
|
||||
jde.Provider.ShouldBe(ConnectionProvider.Oracle);
|
||||
jde.Host.ShouldBe("jde-server");
|
||||
jde.Port.ShouldBe(1521);
|
||||
jde.ServiceName.ShouldBe("JDEPROD");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Deserialize_SqlServerConnection_ParsesAllFields()
|
||||
{
|
||||
// Arrange
|
||||
var json = """
|
||||
{
|
||||
"TestDb": "Server=myserver;Database=TestDB;User Id=testuser;Password=testpass;Encrypt=True;TrustServerCertificate=True;Connection Timeout=60;Application Name=TestApp"
|
||||
}
|
||||
""";
|
||||
|
||||
// Act
|
||||
var section = JsonSerializer.Deserialize<ConnectionStringsSection>(json, JsonOptions);
|
||||
|
||||
// Assert
|
||||
section.ShouldNotBeNull();
|
||||
section.Entries.Count.ShouldBe(1);
|
||||
|
||||
var entry = section.Entries[0];
|
||||
entry.Name.ShouldBe("TestDb");
|
||||
entry.Provider.ShouldBe(ConnectionProvider.SqlServer);
|
||||
entry.Server.ShouldBe("myserver");
|
||||
entry.Database.ShouldBe("TestDB");
|
||||
entry.UserId.ShouldBe("testuser");
|
||||
entry.Password.ShouldBe("testpass");
|
||||
entry.Encrypt.ShouldBe("True");
|
||||
entry.TrustServerCertificate.ShouldBeTrue();
|
||||
entry.ConnectionTimeout.ShouldBe(60);
|
||||
entry.ApplicationName.ShouldBe("TestApp");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Deserialize_OracleConnection_ParsesHostPortService()
|
||||
{
|
||||
// Arrange
|
||||
var json = """
|
||||
{
|
||||
"Oracle": "Data Source=//db-host:1523/PRODDB;User Id=orauser;Password=orapass"
|
||||
}
|
||||
""";
|
||||
|
||||
// Act
|
||||
var section = JsonSerializer.Deserialize<ConnectionStringsSection>(json, JsonOptions);
|
||||
|
||||
// Assert
|
||||
section.ShouldNotBeNull();
|
||||
section.Entries.Count.ShouldBe(1);
|
||||
|
||||
var entry = section.Entries[0];
|
||||
entry.Name.ShouldBe("Oracle");
|
||||
entry.Provider.ShouldBe(ConnectionProvider.Oracle);
|
||||
entry.Host.ShouldBe("db-host");
|
||||
entry.Port.ShouldBe(1523);
|
||||
entry.ServiceName.ShouldBe("PRODDB");
|
||||
entry.UserId.ShouldBe("orauser");
|
||||
entry.Password.ShouldBe("orapass");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Deserialize_EmptyObject_ReturnsEmptyEntries()
|
||||
{
|
||||
// Arrange
|
||||
var json = "{}";
|
||||
|
||||
// Act
|
||||
var section = JsonSerializer.Deserialize<ConnectionStringsSection>(json, JsonOptions);
|
||||
|
||||
// Assert
|
||||
section.ShouldNotBeNull();
|
||||
section.Entries.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Serialize_ToStandardDictionaryFormat()
|
||||
{
|
||||
// Arrange
|
||||
var section = new ConnectionStringsSection
|
||||
{
|
||||
Entries = new List<ConnectionStringEntry>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Name = "TestDb",
|
||||
Provider = ConnectionProvider.SqlServer,
|
||||
Server = "myserver",
|
||||
Database = "TestDB",
|
||||
UserId = "testuser",
|
||||
Password = "testpass"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var json = JsonSerializer.Serialize(section, JsonOptions);
|
||||
|
||||
// Assert
|
||||
json.ShouldNotBeNull();
|
||||
json.ShouldContain("\"TestDb\"");
|
||||
json.ShouldContain("Server=myserver");
|
||||
json.ShouldContain("Database=TestDB");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RoundTrip_PreservesConnections()
|
||||
{
|
||||
// Arrange
|
||||
var original = new ConnectionStringsSection
|
||||
{
|
||||
Entries = new List<ConnectionStringEntry>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Name = "Primary",
|
||||
Provider = ConnectionProvider.SqlServer,
|
||||
Server = "server1",
|
||||
Database = "DB1",
|
||||
UserId = "user1",
|
||||
Password = "pass1"
|
||||
},
|
||||
new()
|
||||
{
|
||||
Name = "Secondary",
|
||||
Provider = ConnectionProvider.Oracle,
|
||||
Host = "oracle-host",
|
||||
Port = 1521,
|
||||
ServiceName = "ORCL",
|
||||
UserId = "user2",
|
||||
Password = "pass2"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var json = JsonSerializer.Serialize(original, JsonOptions);
|
||||
var deserialized = JsonSerializer.Deserialize<ConnectionStringsSection>(json, JsonOptions);
|
||||
|
||||
// Assert
|
||||
deserialized.ShouldNotBeNull();
|
||||
deserialized.Entries.Count.ShouldBe(2);
|
||||
|
||||
var primary = deserialized.Entries.First(e => e.Name == "Primary");
|
||||
primary.Provider.ShouldBe(ConnectionProvider.SqlServer);
|
||||
primary.Server.ShouldBe("server1");
|
||||
primary.Database.ShouldBe("DB1");
|
||||
|
||||
var secondary = deserialized.Entries.First(e => e.Name == "Secondary");
|
||||
secondary.Provider.ShouldBe(ConnectionProvider.Oracle);
|
||||
secondary.Host.ShouldBe("oracle-host");
|
||||
secondary.Port.ShouldBe(1521);
|
||||
secondary.ServiceName.ShouldBe("ORCL");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataAccess.Services;
|
||||
using JdeScoping.Domain.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace JdeScoping.DataAccess.Tests.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for ManualSyncRequestService.
|
||||
/// Tests constructor validation, interface contract compliance, and static helper methods.
|
||||
/// Note: Since this service uses Dapper with raw SQL, full integration tests with
|
||||
/// an actual database are required for complete coverage of the SQL operations.
|
||||
/// </summary>
|
||||
public class ManualSyncRequestServiceTests
|
||||
{
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly ILogger<ManualSyncRequestService> _logger;
|
||||
|
||||
public ManualSyncRequestServiceTests()
|
||||
{
|
||||
_connectionFactory = Substitute.For<IDbConnectionFactory>();
|
||||
_logger = NullLogger<ManualSyncRequestService>.Instance;
|
||||
}
|
||||
|
||||
#region Constructor Tests
|
||||
|
||||
[Fact]
|
||||
public void Constructor_WithNullConnectionFactory_ThrowsArgumentNullException()
|
||||
{
|
||||
// Act & Assert
|
||||
var exception = Should.Throw<ArgumentNullException>(() =>
|
||||
new ManualSyncRequestService(null!, _logger));
|
||||
|
||||
exception.ParamName.ShouldBe("connectionFactory");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Constructor_WithNullLogger_ThrowsArgumentNullException()
|
||||
{
|
||||
// Act & Assert
|
||||
var exception = Should.Throw<ArgumentNullException>(() =>
|
||||
new ManualSyncRequestService(_connectionFactory, null!));
|
||||
|
||||
exception.ParamName.ShouldBe("logger");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Constructor_WithValidDependencies_CreatesInstance()
|
||||
{
|
||||
// Act
|
||||
var service = new ManualSyncRequestService(_connectionFactory, _logger);
|
||||
|
||||
// Assert
|
||||
service.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Interface Contract Tests
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequestService_ImplementsIManualSyncRequestService()
|
||||
{
|
||||
// Arrange & Act
|
||||
var service = new ManualSyncRequestService(_connectionFactory, _logger);
|
||||
|
||||
// Assert
|
||||
service.ShouldBeAssignableTo<IManualSyncRequestService>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetRequestsAsync_HasCorrectSignature()
|
||||
{
|
||||
// Arrange
|
||||
var service = new ManualSyncRequestService(_connectionFactory, _logger);
|
||||
|
||||
// Act - Verify method exists with correct return type
|
||||
var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.GetRequestsAsync));
|
||||
|
||||
// Assert
|
||||
methodInfo.ShouldNotBeNull();
|
||||
methodInfo.ReturnType.ShouldBe(typeof(Task<IReadOnlyList<ManualSyncRequest>>));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetNextPendingRequestAsync_HasCorrectSignature()
|
||||
{
|
||||
// Arrange
|
||||
var service = new ManualSyncRequestService(_connectionFactory, _logger);
|
||||
|
||||
// Act - Verify method exists with correct return type
|
||||
var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.GetNextPendingRequestAsync));
|
||||
|
||||
// Assert
|
||||
methodInfo.ShouldNotBeNull();
|
||||
methodInfo.ReturnType.ShouldBe(typeof(Task<ManualSyncRequest?>));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateRequestAsync_HasCorrectSignature()
|
||||
{
|
||||
// Arrange
|
||||
var service = new ManualSyncRequestService(_connectionFactory, _logger);
|
||||
|
||||
// Act - Verify method exists with correct return type
|
||||
var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.CreateRequestAsync));
|
||||
|
||||
// Assert
|
||||
methodInfo.ShouldNotBeNull();
|
||||
methodInfo.ReturnType.ShouldBe(typeof(Task<ManualSyncRequest>));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CancelRequestAsync_HasCorrectSignature()
|
||||
{
|
||||
// Arrange
|
||||
var service = new ManualSyncRequestService(_connectionFactory, _logger);
|
||||
|
||||
// Act - Verify method exists with correct return type
|
||||
var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.CancelRequestAsync));
|
||||
|
||||
// Assert
|
||||
methodInfo.ShouldNotBeNull();
|
||||
methodInfo.ReturnType.ShouldBe(typeof(Task<bool>));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CompleteRequestAsync_HasCorrectSignature()
|
||||
{
|
||||
// Arrange
|
||||
var service = new ManualSyncRequestService(_connectionFactory, _logger);
|
||||
|
||||
// Act - Verify method exists with correct return type
|
||||
var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.CompleteRequestAsync));
|
||||
|
||||
// Assert
|
||||
methodInfo.ShouldNotBeNull();
|
||||
methodInfo.ReturnType.ShouldBe(typeof(Task<bool>));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Domain Model Tests
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequest_Status_WhenPending_ReturnsPending()
|
||||
{
|
||||
// Arrange
|
||||
var request = new ManualSyncRequest
|
||||
{
|
||||
Id = 1,
|
||||
PipelineName = "TestPipeline",
|
||||
SyncType = "mass",
|
||||
RequestedBy = "testuser",
|
||||
RequestDT = DateTime.UtcNow,
|
||||
CompletedDT = null,
|
||||
CancelDT = null
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
request.Status.ShouldBe("Pending");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequest_Status_WhenCompleted_ReturnsCompleted()
|
||||
{
|
||||
// Arrange
|
||||
var request = new ManualSyncRequest
|
||||
{
|
||||
Id = 1,
|
||||
PipelineName = "TestPipeline",
|
||||
SyncType = "mass",
|
||||
RequestedBy = "testuser",
|
||||
RequestDT = DateTime.UtcNow.AddHours(-1),
|
||||
CompletedDT = DateTime.UtcNow,
|
||||
CancelDT = null
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
request.Status.ShouldBe("Completed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequest_Status_WhenCancelled_ReturnsCancelled()
|
||||
{
|
||||
// Arrange
|
||||
var request = new ManualSyncRequest
|
||||
{
|
||||
Id = 1,
|
||||
PipelineName = "TestPipeline",
|
||||
SyncType = "mass",
|
||||
RequestedBy = "testuser",
|
||||
RequestDT = DateTime.UtcNow.AddHours(-1),
|
||||
CompletedDT = null,
|
||||
CancelDT = DateTime.UtcNow,
|
||||
CancelledBy = "admin"
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
request.Status.ShouldBe("Cancelled");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequest_Status_WhenCancelledAndCompleted_ReturnsCancelled()
|
||||
{
|
||||
// Arrange - Edge case: both CancelDT and CompletedDT are set
|
||||
// Based on the implementation, CancelDT takes precedence
|
||||
var request = new ManualSyncRequest
|
||||
{
|
||||
Id = 1,
|
||||
PipelineName = "TestPipeline",
|
||||
SyncType = "mass",
|
||||
RequestedBy = "testuser",
|
||||
RequestDT = DateTime.UtcNow.AddHours(-2),
|
||||
CompletedDT = DateTime.UtcNow.AddHours(-1),
|
||||
CancelDT = DateTime.UtcNow,
|
||||
CancelledBy = "admin"
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
// CancelDT is checked first in the Status property, so it should return "Cancelled"
|
||||
request.Status.ShouldBe("Cancelled");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequest_DefaultRowVersion_IsEmptyArray()
|
||||
{
|
||||
// Arrange
|
||||
var request = new ManualSyncRequest();
|
||||
|
||||
// Act & Assert
|
||||
request.RowVersion.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequest_DefaultPipelineName_IsEmptyString()
|
||||
{
|
||||
// Arrange
|
||||
var request = new ManualSyncRequest();
|
||||
|
||||
// Act & Assert
|
||||
request.PipelineName.ShouldBe(string.Empty);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequest_DefaultSyncType_IsEmptyString()
|
||||
{
|
||||
// Arrange
|
||||
var request = new ManualSyncRequest();
|
||||
|
||||
// Act & Assert
|
||||
request.SyncType.ShouldBe(string.Empty);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequest_DefaultRequestedBy_IsEmptyString()
|
||||
{
|
||||
// Arrange
|
||||
var request = new ManualSyncRequest();
|
||||
|
||||
// Act & Assert
|
||||
request.RequestedBy.ShouldBe(string.Empty);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManualSyncRequest_CancelledBy_IsNullableAndDefaultsToNull()
|
||||
{
|
||||
// Arrange
|
||||
var request = new ManualSyncRequest();
|
||||
|
||||
// Act & Assert
|
||||
request.CancelledBy.ShouldBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Method Parameter Tests
|
||||
|
||||
[Fact]
|
||||
public void GetRequestsAsync_PendingOnlyParameter_DefaultsToFalse()
|
||||
{
|
||||
// Verify the interface defines correct default parameter
|
||||
var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.GetRequestsAsync));
|
||||
var parameters = methodInfo!.GetParameters();
|
||||
|
||||
// The pendingOnly parameter should have a default value of false
|
||||
var pendingOnlyParam = parameters.FirstOrDefault(p => p.Name == "pendingOnly");
|
||||
pendingOnlyParam.ShouldNotBeNull();
|
||||
pendingOnlyParam.HasDefaultValue.ShouldBeTrue();
|
||||
pendingOnlyParam.DefaultValue.ShouldBe(false);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateRequestAsync_RequiresPipelineName()
|
||||
{
|
||||
// Verify the method has a pipelineName parameter
|
||||
var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CreateRequestAsync));
|
||||
var parameters = methodInfo!.GetParameters();
|
||||
|
||||
var param = parameters.FirstOrDefault(p => p.Name == "pipelineName");
|
||||
param.ShouldNotBeNull();
|
||||
param.ParameterType.ShouldBe(typeof(string));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateRequestAsync_RequiresSyncType()
|
||||
{
|
||||
// Verify the method has a syncType parameter
|
||||
var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CreateRequestAsync));
|
||||
var parameters = methodInfo!.GetParameters();
|
||||
|
||||
var param = parameters.FirstOrDefault(p => p.Name == "syncType");
|
||||
param.ShouldNotBeNull();
|
||||
param.ParameterType.ShouldBe(typeof(string));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateRequestAsync_RequiresRequestedBy()
|
||||
{
|
||||
// Verify the method has a requestedBy parameter
|
||||
var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CreateRequestAsync));
|
||||
var parameters = methodInfo!.GetParameters();
|
||||
|
||||
var param = parameters.FirstOrDefault(p => p.Name == "requestedBy");
|
||||
param.ShouldNotBeNull();
|
||||
param.ParameterType.ShouldBe(typeof(string));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CancelRequestAsync_RequiresId()
|
||||
{
|
||||
// Verify the method has an id parameter
|
||||
var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CancelRequestAsync));
|
||||
var parameters = methodInfo!.GetParameters();
|
||||
|
||||
var param = parameters.FirstOrDefault(p => p.Name == "id");
|
||||
param.ShouldNotBeNull();
|
||||
param.ParameterType.ShouldBe(typeof(int));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CancelRequestAsync_RequiresRowVersion()
|
||||
{
|
||||
// Verify the method has a rowVersion parameter for optimistic concurrency
|
||||
var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CancelRequestAsync));
|
||||
var parameters = methodInfo!.GetParameters();
|
||||
|
||||
var param = parameters.FirstOrDefault(p => p.Name == "rowVersion");
|
||||
param.ShouldNotBeNull();
|
||||
param.ParameterType.ShouldBe(typeof(byte[]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CompleteRequestAsync_RequiresId()
|
||||
{
|
||||
// Verify the method has an id parameter
|
||||
var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CompleteRequestAsync));
|
||||
var parameters = methodInfo!.GetParameters();
|
||||
|
||||
var param = parameters.FirstOrDefault(p => p.Name == "id");
|
||||
param.ShouldNotBeNull();
|
||||
param.ParameterType.ShouldBe(typeof(int));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CompleteRequestAsync_RequiresRowVersion()
|
||||
{
|
||||
// Verify the method has a rowVersion parameter for optimistic concurrency
|
||||
var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CompleteRequestAsync));
|
||||
var parameters = methodInfo!.GetParameters();
|
||||
|
||||
var param = parameters.FirstOrDefault(p => p.Name == "rowVersion");
|
||||
param.ShouldNotBeNull();
|
||||
param.ParameterType.ShouldBe(typeof(byte[]));
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using JdeScoping.Core.Models;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.Core.Models.Infrastructure;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Services;
|
||||
@@ -17,19 +18,28 @@ namespace JdeScoping.DataSync.Tests;
|
||||
public class ScheduleCheckerTests
|
||||
{
|
||||
private readonly IDataUpdateRepository _repository;
|
||||
private readonly IPipelineRegistry _pipelineRegistry;
|
||||
private readonly IOptions<DataSyncOptions> _options;
|
||||
private readonly List<EtlPipelineConfig> _pipelines;
|
||||
private readonly ScheduleChecker _sut;
|
||||
|
||||
public ScheduleCheckerTests()
|
||||
{
|
||||
_repository = Substitute.For<IDataUpdateRepository>();
|
||||
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
|
||||
_pipelines = [];
|
||||
_options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
|
||||
{
|
||||
LookbackMultiplier = 3,
|
||||
DataSources = []
|
||||
});
|
||||
|
||||
// Setup pipeline registry to return our pipeline list
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(_ => _pipelines);
|
||||
|
||||
_sut = new ScheduleChecker(
|
||||
_repository,
|
||||
_pipelineRegistry,
|
||||
_options,
|
||||
NullLogger<ScheduleChecker>.Instance);
|
||||
}
|
||||
@@ -40,8 +50,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenMassNeverRun_ReturnsMassTask()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder", massEnabled: true, dailyEnabled: true, hourlyEnabled: true);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
|
||||
@@ -59,10 +69,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenMassDue_ReturnsMassOverDaily()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 60,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 60, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-120), success: true);
|
||||
@@ -85,11 +93,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenMassNotDue_ChecksDailyAndHourly()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080, // weekly
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true);
|
||||
@@ -114,11 +119,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenDailyDue_ReturnsDailyOverHourly()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -145,11 +147,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenOnlyHourlyDue_ReturnsHourly()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -181,10 +180,8 @@ public class ScheduleCheckerTests
|
||||
{
|
||||
// Arrange: LookbackMultiplier = 3, daily interval = 1440 min
|
||||
// MinimumDT = lastDaily.EndDT - (3 * 1440) = lastDaily.EndDT - 4320 min = 3 days before lastDaily
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true);
|
||||
@@ -214,11 +211,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_HourlySync_UsesHourlyTimestampForMinimumDT()
|
||||
{
|
||||
// Arrange: Hourly uses its own timestamp and interval for MinimumDT calculation
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -251,10 +245,8 @@ public class ScheduleCheckerTests
|
||||
{
|
||||
// Arrange: Test with multiplier = 5
|
||||
_options.Value.LookbackMultiplier = 5;
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true);
|
||||
@@ -277,92 +269,15 @@ public class ScheduleCheckerTests
|
||||
|
||||
#endregion
|
||||
|
||||
#region Disabled Table Handling
|
||||
#region Manual-Only Pipelines
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_DisabledDataSource_ReturnsNoTasks()
|
||||
public async Task GetPendingTasksAsync_ManualOnlyPipeline_ReturnsNoTasks()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder", massEnabled: true);
|
||||
config.IsEnabled = false;
|
||||
_options.Value.DataSources.Add(config);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
|
||||
// Act
|
||||
var tasks = await _sut.GetPendingTasksAsync();
|
||||
|
||||
// Assert
|
||||
tasks.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_DisabledMassSchedule_SkipsMass()
|
||||
{
|
||||
// Arrange: Mass disabled, Daily enabled
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: false,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
// Even with no mass ever run, if mass is disabled, should NOT require mass first
|
||||
// However, current logic requires mass before daily, so this tests that properly
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>
|
||||
{
|
||||
{ "WorkOrder_3", lastMass }
|
||||
});
|
||||
|
||||
// Act
|
||||
var tasks = await _sut.GetPendingTasksAsync();
|
||||
|
||||
// Assert: Should return Daily since mass is disabled but already ran before
|
||||
tasks.ShouldHaveSingleItem();
|
||||
tasks[0].UpdateType.ShouldBe(UpdateTypes.Daily);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_DisabledDailySchedule_SkipsDaily()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: false,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
var lastHourly = CreateDataUpdate("WorkOrder", UpdateTypes.Hourly, now.AddHours(-2), success: true);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>
|
||||
{
|
||||
{ "WorkOrder_3", lastMass },
|
||||
{ "WorkOrder_1", lastHourly }
|
||||
});
|
||||
|
||||
// Act
|
||||
var tasks = await _sut.GetPendingTasksAsync();
|
||||
|
||||
// Assert: Should return Hourly, skipping Daily
|
||||
tasks.ShouldHaveSingleItem();
|
||||
tasks[0].UpdateType.ShouldBe(UpdateTypes.Hourly);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_AllSchedulesDisabled_ReturnsNoTasks()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: false,
|
||||
dailyEnabled: false,
|
||||
hourlyEnabled: false);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080);
|
||||
pipeline.IsManualOnly = true;
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
@@ -382,11 +297,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_NoPriorUpdates_RequiresMassFirst()
|
||||
{
|
||||
// Arrange: Never synced before, all schedules enabled
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
@@ -404,10 +316,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_OnlyMassCompleted_DailyHasNullMinimumDT()
|
||||
{
|
||||
// Arrange: Mass completed, no daily yet
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true);
|
||||
@@ -431,11 +341,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_NeverHadMass_DoesNotReturnDailyOrHourly()
|
||||
{
|
||||
// Arrange: Daily and Hourly enabled but no Mass ever run
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
@@ -456,9 +363,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_FailedMass_ReturnsMassAgain()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-5), success: false);
|
||||
@@ -481,10 +387,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_FailedDaily_ReturnsDailyAgain()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -509,11 +413,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_FailedHourly_ReturnsHourlyAgain()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -544,10 +445,10 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_MultipleTables_ReturnsTasksForEach()
|
||||
{
|
||||
// Arrange
|
||||
var config1 = CreateDataSourceConfig("WorkOrder", massEnabled: true, massInterval: 60);
|
||||
var config2 = CreateDataSourceConfig("LotUsage", massEnabled: true, massInterval: 60);
|
||||
_options.Value.DataSources.Add(config1);
|
||||
_options.Value.DataSources.Add(config2);
|
||||
var pipeline1 = CreatePipeline("WorkOrder", massInterval: 60);
|
||||
var pipeline2 = CreatePipeline("LotUsage", massInterval: 60);
|
||||
_pipelines.Add(pipeline1);
|
||||
_pipelines.Add(pipeline2);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
@@ -565,13 +466,10 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_MultipleTables_DifferentSchedulesDue()
|
||||
{
|
||||
// Arrange
|
||||
var config1 = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
var config2 = CreateDataSourceConfig("LotUsage",
|
||||
massEnabled: true, massInterval: 60);
|
||||
_options.Value.DataSources.Add(config1);
|
||||
_options.Value.DataSources.Add(config2);
|
||||
var pipeline1 = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
var pipeline2 = CreatePipeline("LotUsage", massInterval: 60);
|
||||
_pipelines.Add(pipeline1);
|
||||
_pipelines.Add(pipeline2);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMassWorkOrder = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -603,11 +501,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_NothingDue_ReturnsEmptyList()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
// All syncs completed recently
|
||||
@@ -631,9 +526,9 @@ public class ScheduleCheckerTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_NoDataSources_ReturnsEmptyList()
|
||||
public async Task GetPendingTasksAsync_NoPipelines_ReturnsEmptyList()
|
||||
{
|
||||
// Arrange: No data sources configured
|
||||
// Arrange: No pipelines configured
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
|
||||
@@ -648,35 +543,29 @@ public class ScheduleCheckerTests
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static DataSourceConfig CreateDataSourceConfig(
|
||||
string tableName,
|
||||
bool massEnabled = false,
|
||||
int massInterval = 10080,
|
||||
bool dailyEnabled = false,
|
||||
int dailyInterval = 1440,
|
||||
bool hourlyEnabled = false,
|
||||
int hourlyInterval = 60)
|
||||
private static EtlPipelineConfig CreatePipeline(
|
||||
string name,
|
||||
int? massInterval = null,
|
||||
int? dailyInterval = null,
|
||||
int? hourlyInterval = null)
|
||||
{
|
||||
return new DataSourceConfig
|
||||
return new EtlPipelineConfig
|
||||
{
|
||||
TableName = tableName,
|
||||
SourceSystem = "JDE",
|
||||
SourceData = tableName.ToUpper(),
|
||||
Name = name,
|
||||
IsEnabled = true,
|
||||
MassConfig = new ScheduleConfig
|
||||
IsManualOnly = false,
|
||||
MassSyncIntervalMinutes = massInterval,
|
||||
DailySyncIntervalMinutes = dailyInterval,
|
||||
HourlySyncIntervalMinutes = hourlyInterval,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Enabled = massEnabled,
|
||||
IntervalMinutes = massInterval
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM Test"
|
||||
},
|
||||
DailyConfig = new ScheduleConfig
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Enabled = dailyEnabled,
|
||||
IntervalMinutes = dailyInterval
|
||||
},
|
||||
HourlyConfig = new ScheduleConfig
|
||||
{
|
||||
Enabled = hourlyEnabled,
|
||||
IntervalMinutes = hourlyInterval
|
||||
Table = name,
|
||||
MatchColumns = ["Id"]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
using System.Text.Json;
|
||||
using Xunit;
|
||||
using MsOptions = Microsoft.Extensions.Options.Options;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Services;
|
||||
|
||||
public class PipelineRegistryTests : IDisposable
|
||||
{
|
||||
private readonly string _testDirectory;
|
||||
private readonly IPipelineValidator _validator;
|
||||
private readonly IOptions<DataSyncOptions> _options;
|
||||
private readonly ILogger<PipelineRegistry> _logger;
|
||||
private readonly IHostEnvironment _environment;
|
||||
|
||||
public PipelineRegistryTests()
|
||||
{
|
||||
_testDirectory = Path.Combine(Path.GetTempPath(), $"PipelineRegistryTests_{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_testDirectory);
|
||||
|
||||
_validator = new PipelineValidator();
|
||||
_logger = Substitute.For<ILogger<PipelineRegistry>>();
|
||||
_environment = Substitute.For<IHostEnvironment>();
|
||||
_environment.ContentRootPath.Returns(_testDirectory);
|
||||
|
||||
_options = MsOptions.Create(new DataSyncOptions
|
||||
{
|
||||
PipelinesDirectory = "Pipelines",
|
||||
StrictPipelineValidation = false
|
||||
});
|
||||
|
||||
// Create the Pipelines subdirectory
|
||||
Directory.CreateDirectory(Path.Combine(_testDirectory, "Pipelines"));
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_testDirectory))
|
||||
{
|
||||
Directory.Delete(_testDirectory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private string PipelinesDir => Path.Combine(_testDirectory, "Pipelines");
|
||||
|
||||
#region Loading Tests
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_ValidDirectory_LoadsAll()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
CreatePipelineFile("Pipeline2", true);
|
||||
CreatePipelineFile("Pipeline3", true);
|
||||
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.Success.ShouldBeTrue();
|
||||
result.PipelinesLoaded.ShouldBe(3);
|
||||
registry.GetAllPipelines().Count.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_EmptyDirectory_ReturnsEmpty()
|
||||
{
|
||||
// Arrange - empty directory (Pipelines subdirectory is already created but empty)
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.Success.ShouldBeTrue();
|
||||
result.PipelinesLoaded.ShouldBe(0);
|
||||
registry.GetAllPipelines().ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_OnlyLoadsJsonFiles()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("ValidPipeline", true);
|
||||
File.WriteAllText(Path.Combine(PipelinesDir, "readme.txt"), "Some text");
|
||||
File.WriteAllText(Path.Combine(PipelinesDir, "config.xml"), "<xml/>");
|
||||
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.PipelinesLoaded.ShouldBe(1);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Error Handling Tests
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_InvalidJson_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
File.WriteAllText(Path.Combine(PipelinesDir, "pipeline.Invalid.json"), "{ invalid json }");
|
||||
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.Errors.ShouldContain(e => e.ErrorType.Contains("parse", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_DuplicateNames_ReturnsError()
|
||||
{
|
||||
// Arrange - two files with same pipeline name
|
||||
CreatePipelineFile("DuplicateName", true, "pipeline.First.json");
|
||||
CreatePipelineFile("DuplicateName", true, "pipeline.Second.json");
|
||||
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.Errors.ShouldContain(e => e.ErrorType.Contains("validation", StringComparison.OrdinalIgnoreCase)
|
||||
&& e.Messages.Any(m => m.Contains("Duplicate", StringComparison.OrdinalIgnoreCase)));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Retrieval Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetPipeline_ByName_ReturnsCorrect()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("TestPipeline", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var pipeline = registry.GetPipeline("TestPipeline");
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.Name.ShouldBe("TestPipeline");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPipeline_CaseInsensitive_ReturnsCorrect()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("MyPipeline", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var pipeline = registry.GetPipeline("mypipeline");
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.Name.ShouldBe("MyPipeline");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPipeline_NotFound_ReturnsNull()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("ExistingPipeline", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var pipeline = registry.GetPipeline("NonExistentPipeline");
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetEnabledPipelines_OnlyReturnsEnabled()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("EnabledPipeline1", true);
|
||||
CreatePipelineFile("EnabledPipeline2", true);
|
||||
CreatePipelineFile("DisabledPipeline", false);
|
||||
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var enabledPipelines = registry.GetEnabledPipelines();
|
||||
|
||||
// Assert
|
||||
enabledPipelines.Count.ShouldBe(2);
|
||||
enabledPipelines.ShouldAllBe(p => p.IsEnabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAllPipelines_IncludesDisabled()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("EnabledPipeline", true);
|
||||
CreatePipelineFile("DisabledPipeline", false);
|
||||
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var allPipelines = registry.GetAllPipelines();
|
||||
|
||||
// Assert
|
||||
allPipelines.Count.ShouldBe(2);
|
||||
allPipelines.ShouldContain(p => !p.IsEnabled);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Reload Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadAsync_UpdatesSnapshot()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Add another pipeline
|
||||
CreatePipelineFile("Pipeline2", true);
|
||||
|
||||
// Act
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
registry.GetAllPipelines().Count.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadAsync_IncrementsVersion()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
await registry.ReloadAsync();
|
||||
var version1 = registry.Version;
|
||||
|
||||
await registry.ReloadAsync();
|
||||
var version2 = registry.Version;
|
||||
|
||||
// Assert
|
||||
version2.ShouldBe(version1 + 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadAsync_UpdatesLastLoadedAt()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var before = DateTime.UtcNow;
|
||||
await registry.ReloadAsync();
|
||||
var after = DateTime.UtcNow;
|
||||
|
||||
// Assert
|
||||
registry.LastLoadedAt.ShouldNotBeNull();
|
||||
registry.LastLoadedAt.Value.ShouldBeGreaterThanOrEqualTo(before.AddSeconds(-1));
|
||||
registry.LastLoadedAt.Value.ShouldBeLessThanOrEqualTo(after.AddSeconds(1));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region IsValidPipelineAndSyncType Tests
|
||||
|
||||
[Fact]
|
||||
public async Task IsValidPipelineAndSyncType_ValidCombination_ReturnsTrue()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("TestPipeline", true, massSyncInterval: 1440, dailySyncInterval: 60);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act & Assert
|
||||
registry.IsValidPipelineAndSyncType("TestPipeline", "mass").ShouldBeTrue();
|
||||
registry.IsValidPipelineAndSyncType("TestPipeline", "daily").ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsValidPipelineAndSyncType_UnsupportedSyncType_ReturnsFalse()
|
||||
{
|
||||
// Arrange - only mass sync
|
||||
CreatePipelineFile("TestPipeline", true, massSyncInterval: 1440);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act & Assert
|
||||
registry.IsValidPipelineAndSyncType("TestPipeline", "hourly").ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsValidPipelineAndSyncType_UnknownPipeline_ReturnsFalse()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("TestPipeline", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act & Assert
|
||||
registry.IsValidPipelineAndSyncType("UnknownPipeline", "mass").ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsValidPipelineAndSyncType_DisabledPipeline_ReturnsFalse()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("DisabledPipeline", false, massSyncInterval: 1440);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act & Assert
|
||||
registry.IsValidPipelineAndSyncType("DisabledPipeline", "mass").ShouldBeFalse();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Thread Safety Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ReadOperations_ThreadSafe()
|
||||
{
|
||||
// Arrange
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
CreatePipelineFile($"Pipeline{i}", true);
|
||||
}
|
||||
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act - multiple concurrent reads
|
||||
var tasks = Enumerable.Range(0, 100).Select(_ => Task.Run(() =>
|
||||
{
|
||||
var all = registry.GetAllPipelines();
|
||||
var enabled = registry.GetEnabledPipelines();
|
||||
var specific = registry.GetPipeline("Pipeline5");
|
||||
return all.Count + enabled.Count + (specific != null ? 1 : 0);
|
||||
}));
|
||||
|
||||
// Assert - no exceptions
|
||||
var results = await Task.WhenAll(tasks);
|
||||
results.ShouldAllBe(r => r > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentReloads_Serialized()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act - multiple concurrent reloads
|
||||
var tasks = Enumerable.Range(0, 10).Select(_ => registry.ReloadAsync());
|
||||
|
||||
// Assert - no exceptions and final state is valid
|
||||
await Task.WhenAll(tasks);
|
||||
registry.GetAllPipelines().Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private PipelineRegistry CreateRegistry() =>
|
||||
new PipelineRegistry(_options, _validator, _logger, _environment);
|
||||
|
||||
private void CreatePipelineFile(
|
||||
string name,
|
||||
bool isEnabled,
|
||||
string? fileName = null,
|
||||
int? massSyncInterval = 1440,
|
||||
int? dailySyncInterval = null,
|
||||
int? hourlySyncInterval = null)
|
||||
{
|
||||
var pipeline = new
|
||||
{
|
||||
name,
|
||||
isEnabled,
|
||||
isManualOnly = !massSyncInterval.HasValue && !dailySyncInterval.HasValue && !hourlySyncInterval.HasValue && !isEnabled,
|
||||
massSyncIntervalMinutes = massSyncInterval,
|
||||
dailySyncIntervalMinutes = dailySyncInterval,
|
||||
hourlySyncIntervalMinutes = hourlySyncInterval,
|
||||
source = new
|
||||
{
|
||||
connection = "jde",
|
||||
query = "SELECT * FROM TestTable"
|
||||
},
|
||||
destination = new
|
||||
{
|
||||
table = $"{name}_Table",
|
||||
matchColumns = new[] { "Id" }
|
||||
}
|
||||
};
|
||||
|
||||
var json = JsonSerializer.Serialize(pipeline, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = true,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
|
||||
});
|
||||
|
||||
var actualFileName = fileName ?? $"pipeline.{name}.json";
|
||||
var filePath = Path.Combine(PipelinesDir, actualFileName);
|
||||
File.WriteAllText(filePath, json);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,483 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Services;
|
||||
|
||||
public class PipelineValidatorTests
|
||||
{
|
||||
private readonly IPipelineValidator _validator;
|
||||
|
||||
public PipelineValidatorTests()
|
||||
{
|
||||
_validator = new PipelineValidator();
|
||||
}
|
||||
|
||||
#region Name/Filename Matching
|
||||
|
||||
[Fact]
|
||||
public void Validate_NameMatchesFilename_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
result.Errors.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_NameMismatchFilename_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("WrongName");
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.CorrectName.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("does not match filename", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_NameCaseInsensitive_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("testpipeline");
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Source Validation
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingSource_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = null!,
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Source is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingSourceConnection_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "",
|
||||
Query = "SELECT * FROM table"
|
||||
},
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Connection is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_InvalidConnection_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "invalid_db",
|
||||
Query = "SELECT * FROM table"
|
||||
},
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("not valid", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ValidConnections_Pass()
|
||||
{
|
||||
// Arrange & Act & Assert
|
||||
foreach (var connection in new[] { "jde", "cms", "giw", "lotfinder" })
|
||||
{
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.Source.Connection = connection;
|
||||
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
result.IsValid.ShouldBeTrue($"Connection '{connection}' should be valid");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingSourceQuery_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = ""
|
||||
},
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Query is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Destination Validation
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingDestination_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = CreateValidSource(),
|
||||
Destination = null!
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Destination is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingDestinationTable_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = CreateValidSource(),
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Table = "",
|
||||
MatchColumns = ["Id"]
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Table is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_EmptyMatchColumns_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = CreateValidSource(),
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Table = "TestTable",
|
||||
MatchColumns = []
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("MatchColumns", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Interval Validation
|
||||
|
||||
[Fact]
|
||||
public void Validate_EnabledWithoutAnyInterval_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
IsManualOnly = false,
|
||||
// No intervals set
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("At least one sync interval", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ManualOnlyWithoutInterval_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
IsManualOnly = true,
|
||||
// No intervals set - ok for manual-only
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_DisabledWithoutInterval_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = false,
|
||||
// No intervals set - ok for disabled
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ZeroMassInterval_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.MassSyncIntervalMinutes = 0;
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("MassSyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_NegativeInterval_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.DailySyncIntervalMinutes = -60;
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("DailySyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Warning Cases
|
||||
|
||||
[Fact]
|
||||
public void Validate_HourlyWithoutDaily_ReturnsWarning()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
HourlySyncIntervalMinutes = 15,
|
||||
// No daily interval
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue(); // Warnings don't fail validation
|
||||
result.Warnings.ShouldContain(w => w.Contains("daily", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Script Validation
|
||||
|
||||
[Fact]
|
||||
public void Validate_PreScriptWithEmptyScript_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.PreScripts = [new ScriptElement { Script = "" }];
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("PreScripts", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_PostScriptWithEmptyScript_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.PostScripts = [new ScriptElement { Script = "" }];
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("PostScripts", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ValidScripts_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.PreScripts = [new ScriptElement { Script = "TRUNCATE TABLE Staging" }];
|
||||
pipeline.PostScripts = [new ScriptElement { Script = "EXEC ProcessData" }];
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Complete Valid Pipeline
|
||||
|
||||
[Fact]
|
||||
public void Validate_CompleteValidPipeline_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "WorkOrder_Curr",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
DailySyncIntervalMinutes = 60,
|
||||
HourlySyncIntervalMinutes = 15,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM WorkOrders WHERE ModDate > @lastSync",
|
||||
MassQuery = "SELECT * FROM WorkOrders"
|
||||
},
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Table = "WorkOrder_Curr",
|
||||
MatchColumns = ["OrderNumber"]
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.WorkOrder_Curr.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
result.Errors.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static EtlPipelineConfig CreateValidPipeline(string name) => new()
|
||||
{
|
||||
Name = name,
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
private static SourceElement CreateValidSource() => new()
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM TestTable"
|
||||
};
|
||||
|
||||
private static DestinationElement CreateValidDestination() => new()
|
||||
{
|
||||
Table = "TestTable",
|
||||
MatchColumns = ["Id"]
|
||||
};
|
||||
|
||||
#endregion
|
||||
}
|
||||
Reference in New Issue
Block a user