From 29ac56006dba7779e064a2b59274c9eaa78c3f07 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Jan 2026 17:48:33 -0500 Subject: [PATCH] feat: implement ETL pipeline redesign and ConfigManager improvements - Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings --- .../ManualSync/CancelManualSyncRequestDto.cs | 12 + .../ManualSync/CreateManualSyncRequestDto.cs | 17 + .../ManualSync/ManualSyncRequestViewModel.cs | 57 +++ .../ManualSync/PipelineInfoViewModel.cs | 17 + .../Controllers/ManualSyncController.cs | 189 +++++++ .../Controllers/PipelineController.cs | 224 ++++++++ .../DataSync/NewSyncRequestDialog.razor | 47 ++ .../DataSync/NewSyncRequestDialog.razor.cs | 114 +++++ .../JdeScoping.Client/Layout/MainLayout.razor | 1 + .../Pages/DataSync/DataSyncRequests.razor | 78 +++ .../Pages/DataSync/DataSyncRequests.razor.cs | 232 +++++++++ NEW/src/JdeScoping.Client/Program.cs | 2 + .../Services/ManualSyncApiClient.cs | 68 +++ .../Services/PipelineApiClient.cs | 29 ++ .../JdeScoping.Core/ApiContracts/ApiRoutes.cs | 40 ++ .../ApiContracts/IManualSyncApiClient.cs | 42 ++ .../ApiContracts/IPipelineApiClient.cs | 120 +++++ .../ViewModels/CreateManualSyncRequestDto.cs | 17 + .../ViewModels/ManualSyncRequestViewModel.cs | 57 +++ .../ViewModels/PipelineInfoViewModel.cs | 17 + .../DependencyInjection.cs | 3 + .../JdeScoping.DataAccess.csproj | 1 + .../Services/IManualSyncRequestService.cs | 68 +++ .../Services/ManualSyncRequestService.cs | 202 ++++++++ .../Configuration/DestinationElement.cs | 22 + .../Configuration/EtlPipelineConfig.cs | 78 +++ .../Configuration/ParameterElement.cs | 27 + .../Configuration/ScriptElement.cs | 17 + .../Configuration/SourceElement.cs | 27 + .../Configuration/TransformElement.cs | 21 + .../Contracts/ISyncOrchestrator.cs | 10 + .../DependencyInjection.cs | 7 + .../Models/DataUpdateTask.cs | 61 ++- .../Options/DataSyncOptions.cs | 12 + .../Options/WorkProcessorOptions.cs | 8 + .../Services/IPipelineRegistry.cs | 115 +++++ .../Services/IPipelineValidator.cs | 38 ++ .../Services/PipelineRegistry.cs | 263 ++++++++++ .../Services/PipelineRegistryInitializer.cs | 89 ++++ .../Services/PipelineSnapshot.cs | 61 +++ .../Services/PipelineValidator.cs | 168 ++++++ .../Services/ScheduleChecker.cs | 93 ++-- .../Services/SyncOrchestrator.cs | 52 +- NEW/src/JdeScoping.DataSync/WorkProcessor.cs | 181 ++++++- .../060_CreateManualSyncRequestTable.sql | 43 ++ .../JdeScoping.Domain.csproj | 9 + .../Models/ManualSyncRequest.cs | 59 +++ .../Pipelines/pipeline.Branch.json | 24 + .../Pipelines/pipeline.FunctionCode.json | 21 + .../Pipelines/pipeline.Item.json | 24 + .../Pipelines/pipeline.JdeUser.json | 21 + .../Pipelines/pipeline.Lot.json | 24 + .../Pipelines/pipeline.LotUsage_Curr.json | 24 + .../Pipelines/pipeline.MisData_Curr.json | 29 ++ .../Pipelines/pipeline.MisData_Hist.json | 27 + .../Pipelines/pipeline.OrgHierarchy.json | 24 + .../Pipelines/pipeline.ProfitCenter.json | 24 + .../Pipelines/pipeline.RouteMaster.json | 24 + .../Pipelines/pipeline.StatusCode.json | 24 + .../Pipelines/pipeline.WorkCenter.json | 24 + .../pipeline.WorkOrderComponent_Curr.json | 24 + .../Pipelines/pipeline.WorkOrderRouting.json | 24 + .../pipeline.WorkOrderStep_Curr.json | 24 + .../pipeline.WorkOrderTime_Curr.json | 24 + .../Pipelines/pipeline.WorkOrder_Curr.json | 24 + NEW/src/JdeScoping.Host/appsettings.json | 2 + .../Models/ConnectionStringsSection.cs | 4 + .../ConnectionStringsSectionConverter.cs | 293 +++++++++++ .../Forms/PipelineEditorViewModel.cs | 86 +++- .../ViewModels/MainWindowViewModel.cs | 4 +- .../PipelineSteps/SourceStepViewModel.cs | 8 +- .../Views/Editors/SourceEditorView.axaml | 13 +- .../Forms/ConnectionStringsFormView.axaml | 99 ++-- .../Views/Forms/PipelineEditorView.axaml | 49 +- .../Views/MainWindow.axaml | 3 + .../Controllers/ManualSyncControllerTests.cs | 402 +++++++++++++++ .../Controllers/PipelineControllerTests.cs | 289 +++++++++++ .../ConnectionStringsSectionConverterTests.cs | 198 +++++++ .../Services/ManualSyncRequestServiceTests.cs | 382 ++++++++++++++ .../ScheduleCheckerTests.cs | 255 +++------ .../Services/PipelineRegistryTests.cs | 434 ++++++++++++++++ .../Services/PipelineValidatorTests.cs | 483 ++++++++++++++++++ 82 files changed, 6257 insertions(+), 296 deletions(-) create mode 100644 NEW/src/JdeScoping.Api/Contracts/ManualSync/CancelManualSyncRequestDto.cs create mode 100644 NEW/src/JdeScoping.Api/Contracts/ManualSync/CreateManualSyncRequestDto.cs create mode 100644 NEW/src/JdeScoping.Api/Contracts/ManualSync/ManualSyncRequestViewModel.cs create mode 100644 NEW/src/JdeScoping.Api/Contracts/ManualSync/PipelineInfoViewModel.cs create mode 100644 NEW/src/JdeScoping.Api/Controllers/ManualSyncController.cs create mode 100644 NEW/src/JdeScoping.Api/Controllers/PipelineController.cs create mode 100644 NEW/src/JdeScoping.Client/Components/DataSync/NewSyncRequestDialog.razor create mode 100644 NEW/src/JdeScoping.Client/Components/DataSync/NewSyncRequestDialog.razor.cs create mode 100644 NEW/src/JdeScoping.Client/Pages/DataSync/DataSyncRequests.razor create mode 100644 NEW/src/JdeScoping.Client/Pages/DataSync/DataSyncRequests.razor.cs create mode 100644 NEW/src/JdeScoping.Client/Services/ManualSyncApiClient.cs create mode 100644 NEW/src/JdeScoping.Client/Services/PipelineApiClient.cs create mode 100644 NEW/src/JdeScoping.Core/ApiContracts/IManualSyncApiClient.cs create mode 100644 NEW/src/JdeScoping.Core/ApiContracts/IPipelineApiClient.cs create mode 100644 NEW/src/JdeScoping.Core/ViewModels/CreateManualSyncRequestDto.cs create mode 100644 NEW/src/JdeScoping.Core/ViewModels/ManualSyncRequestViewModel.cs create mode 100644 NEW/src/JdeScoping.Core/ViewModels/PipelineInfoViewModel.cs create mode 100644 NEW/src/JdeScoping.DataAccess/Services/IManualSyncRequestService.cs create mode 100644 NEW/src/JdeScoping.DataAccess/Services/ManualSyncRequestService.cs create mode 100644 NEW/src/JdeScoping.DataSync/Configuration/DestinationElement.cs create mode 100644 NEW/src/JdeScoping.DataSync/Configuration/EtlPipelineConfig.cs create mode 100644 NEW/src/JdeScoping.DataSync/Configuration/ParameterElement.cs create mode 100644 NEW/src/JdeScoping.DataSync/Configuration/ScriptElement.cs create mode 100644 NEW/src/JdeScoping.DataSync/Configuration/SourceElement.cs create mode 100644 NEW/src/JdeScoping.DataSync/Configuration/TransformElement.cs create mode 100644 NEW/src/JdeScoping.DataSync/Services/IPipelineRegistry.cs create mode 100644 NEW/src/JdeScoping.DataSync/Services/IPipelineValidator.cs create mode 100644 NEW/src/JdeScoping.DataSync/Services/PipelineRegistry.cs create mode 100644 NEW/src/JdeScoping.DataSync/Services/PipelineRegistryInitializer.cs create mode 100644 NEW/src/JdeScoping.DataSync/Services/PipelineSnapshot.cs create mode 100644 NEW/src/JdeScoping.DataSync/Services/PipelineValidator.cs create mode 100644 NEW/src/JdeScoping.Database/Scripts/060_CreateManualSyncRequestTable.sql create mode 100644 NEW/src/JdeScoping.Domain/JdeScoping.Domain.csproj create mode 100644 NEW/src/JdeScoping.Domain/Models/ManualSyncRequest.cs create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.Branch.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.FunctionCode.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.Item.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.JdeUser.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.Lot.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.LotUsage_Curr.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.MisData_Curr.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.MisData_Hist.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.OrgHierarchy.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.ProfitCenter.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.RouteMaster.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.StatusCode.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkCenter.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderComponent_Curr.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderRouting.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderStep_Curr.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderTime_Curr.json create mode 100644 NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrder_Curr.json create mode 100644 NEW/src/Utils/JdeScoping.ConfigManager/Models/ConnectionStringsSectionConverter.cs create mode 100644 NEW/tests/JdeScoping.Api.Tests/Controllers/ManualSyncControllerTests.cs create mode 100644 NEW/tests/JdeScoping.Api.Tests/Controllers/PipelineControllerTests.cs create mode 100644 NEW/tests/JdeScoping.ConfigManager.Tests/Models/ConnectionStringsSectionConverterTests.cs create mode 100644 NEW/tests/JdeScoping.DataAccess.Tests/Services/ManualSyncRequestServiceTests.cs create mode 100644 NEW/tests/JdeScoping.DataSync.Tests/Services/PipelineRegistryTests.cs create mode 100644 NEW/tests/JdeScoping.DataSync.Tests/Services/PipelineValidatorTests.cs diff --git a/NEW/src/JdeScoping.Api/Contracts/ManualSync/CancelManualSyncRequestDto.cs b/NEW/src/JdeScoping.Api/Contracts/ManualSync/CancelManualSyncRequestDto.cs new file mode 100644 index 0000000..d25fdd8 --- /dev/null +++ b/NEW/src/JdeScoping.Api/Contracts/ManualSync/CancelManualSyncRequestDto.cs @@ -0,0 +1,12 @@ +namespace JdeScoping.Api.Contracts.ManualSync; + +/// +/// Request to cancel a manual sync request. +/// +public class CancelManualSyncRequestDto +{ + /// + /// The row version for optimistic concurrency (Base64 encoded). + /// + public string RowVersionBase64 { get; set; } = string.Empty; +} diff --git a/NEW/src/JdeScoping.Api/Contracts/ManualSync/CreateManualSyncRequestDto.cs b/NEW/src/JdeScoping.Api/Contracts/ManualSync/CreateManualSyncRequestDto.cs new file mode 100644 index 0000000..c026ffd --- /dev/null +++ b/NEW/src/JdeScoping.Api/Contracts/ManualSync/CreateManualSyncRequestDto.cs @@ -0,0 +1,17 @@ +namespace JdeScoping.Api.Contracts.ManualSync; + +/// +/// Request to create a new manual sync request. +/// +public class CreateManualSyncRequestDto +{ + /// + /// The name of the pipeline to sync. + /// + public string PipelineName { get; set; } = string.Empty; + + /// + /// The type of sync to perform (mass, daily, hourly). + /// + public string SyncType { get; set; } = string.Empty; +} diff --git a/NEW/src/JdeScoping.Api/Contracts/ManualSync/ManualSyncRequestViewModel.cs b/NEW/src/JdeScoping.Api/Contracts/ManualSync/ManualSyncRequestViewModel.cs new file mode 100644 index 0000000..dad7066 --- /dev/null +++ b/NEW/src/JdeScoping.Api/Contracts/ManualSync/ManualSyncRequestViewModel.cs @@ -0,0 +1,57 @@ +namespace JdeScoping.Api.Contracts.ManualSync; + +/// +/// View model for a manual sync request. +/// +public class ManualSyncRequestViewModel +{ + /// + /// The unique identifier for the sync request. + /// + public int Id { get; set; } + + /// + /// The name of the pipeline being synced. + /// + public string PipelineName { get; set; } = string.Empty; + + /// + /// The type of sync (mass, daily, hourly). + /// + public string SyncType { get; set; } = string.Empty; + + /// + /// The date and time the request was made. + /// + public DateTime RequestDT { get; set; } + + /// + /// The username of the person who requested the sync. + /// + public string RequestedBy { get; set; } = string.Empty; + + /// + /// The date and time the sync completed, if applicable. + /// + public DateTime? CompletedDT { get; set; } + + /// + /// The date and time the sync was cancelled, if applicable. + /// + public DateTime? CancelDT { get; set; } + + /// + /// The username of the person who cancelled the sync, if applicable. + /// + public string? CancelledBy { get; set; } + + /// + /// The current status of the sync request. + /// + public string Status { get; set; } = string.Empty; + + /// + /// The row version for optimistic concurrency (Base64 encoded). + /// + public string RowVersionBase64 { get; set; } = string.Empty; +} diff --git a/NEW/src/JdeScoping.Api/Contracts/ManualSync/PipelineInfoViewModel.cs b/NEW/src/JdeScoping.Api/Contracts/ManualSync/PipelineInfoViewModel.cs new file mode 100644 index 0000000..3ec1047 --- /dev/null +++ b/NEW/src/JdeScoping.Api/Contracts/ManualSync/PipelineInfoViewModel.cs @@ -0,0 +1,17 @@ +namespace JdeScoping.Api.Contracts.ManualSync; + +/// +/// View model for pipeline information. +/// +public class PipelineInfoViewModel +{ + /// + /// The name of the pipeline. + /// + public string Name { get; set; } = string.Empty; + + /// + /// The sync types supported by this pipeline. + /// + public List SupportedSyncTypes { get; set; } = new(); +} diff --git a/NEW/src/JdeScoping.Api/Controllers/ManualSyncController.cs b/NEW/src/JdeScoping.Api/Controllers/ManualSyncController.cs new file mode 100644 index 0000000..82a59c7 --- /dev/null +++ b/NEW/src/JdeScoping.Api/Controllers/ManualSyncController.cs @@ -0,0 +1,189 @@ +using JdeScoping.Api.Contracts.ManualSync; +using JdeScoping.DataAccess.Services; +using JdeScoping.DataSync.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace JdeScoping.Api.Controllers; + +/// +/// API endpoints for manual data sync request management. +/// +[Route("api/manual-sync")] +[ApiController] +[Authorize] +public class ManualSyncController : ApiControllerBase +{ + private readonly IManualSyncRequestService _manualSyncRequestService; + private readonly IPipelineRegistry _pipelineRegistry; + + /// + /// Initializes a new instance of the class. + /// + /// The service for managing manual sync requests. + /// The pipeline registry for querying pipeline information. + public ManualSyncController( + IManualSyncRequestService manualSyncRequestService, + IPipelineRegistry pipelineRegistry) + { + _manualSyncRequestService = manualSyncRequestService; + _pipelineRegistry = pipelineRegistry; + } + + /// + /// Gets all manual sync requests, optionally filtered to pending only. + /// + /// If true, returns only pending requests. Default is false. + /// Cancellation token. + /// A list of manual sync request view models. + [HttpGet] + [ProducesResponseType(typeof(List), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status401Unauthorized)] + public async Task>> GetRequests( + [FromQuery] bool pendingOnly = false, + CancellationToken ct = default) + { + var requests = await _manualSyncRequestService.GetRequestsAsync(pendingOnly, ct); + + var viewModels = requests.Select(r => new ManualSyncRequestViewModel + { + Id = r.Id, + PipelineName = r.PipelineName, + SyncType = r.SyncType, + RequestDT = r.RequestDT, + RequestedBy = r.RequestedBy, + CompletedDT = r.CompletedDT, + CancelDT = r.CancelDT, + CancelledBy = r.CancelledBy, + Status = r.Status, + RowVersionBase64 = Convert.ToBase64String(r.RowVersion) + }).ToList(); + + return Ok(viewModels); + } + + /// + /// Gets all available pipelines with their supported sync types. + /// + /// A list of pipeline information view models. + [HttpGet("pipelines")] + [ProducesResponseType(typeof(List), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status401Unauthorized)] + public ActionResult> GetPipelines() + { + var pipelines = _pipelineRegistry.GetEnabledPipelines() + .Select(p => new PipelineInfoViewModel + { + Name = p.Name, + SupportedSyncTypes = GetSupportedSyncTypes(p) + }) + .ToList(); + + return Ok(pipelines); + } + + /// + /// Creates a new manual sync request. + /// + /// The create request containing pipeline name and sync type. + /// Cancellation token. + /// The created manual sync request view model. + [HttpPost] + [ProducesResponseType(typeof(ManualSyncRequestViewModel), StatusCodes.Status201Created)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + [ProducesResponseType(StatusCodes.Status401Unauthorized)] + public async Task> CreateRequest( + [FromBody] CreateManualSyncRequestDto dto, + CancellationToken ct = default) + { + var username = User.Identity?.Name; + if (string.IsNullOrEmpty(username)) + { + return Unauthorized(); + } + + // Validate pipeline and sync type combination using the registry + if (!_pipelineRegistry.IsValidPipelineAndSyncType(dto.PipelineName, dto.SyncType)) + { + return BadRequest($"Invalid pipeline/sync type combination. Pipeline '{dto.PipelineName}' does not support sync type '{dto.SyncType}'."); + } + + var request = await _manualSyncRequestService.CreateRequestAsync( + dto.PipelineName, + dto.SyncType, + username, + ct); + + var viewModel = new ManualSyncRequestViewModel + { + Id = request.Id, + PipelineName = request.PipelineName, + SyncType = request.SyncType, + RequestDT = request.RequestDT, + RequestedBy = request.RequestedBy, + CompletedDT = request.CompletedDT, + CancelDT = request.CancelDT, + CancelledBy = request.CancelledBy, + Status = request.Status, + RowVersionBase64 = Convert.ToBase64String(request.RowVersion) + }; + + return CreatedAtAction(nameof(GetRequests), viewModel); + } + + /// + /// Cancels a pending manual sync request. + /// + /// The ID of the request to cancel. + /// The cancel request containing the row version for concurrency. + /// Cancellation token. + /// A success message if cancelled, or a conflict error if already processed. + [HttpPost("{id:int}/cancel")] + [ProducesResponseType(typeof(object), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status401Unauthorized)] + [ProducesResponseType(StatusCodes.Status409Conflict)] + public async Task CancelRequest( + int id, + [FromBody] CancelManualSyncRequestDto dto, + CancellationToken ct = default) + { + var username = User.Identity?.Name; + if (string.IsNullOrEmpty(username)) + { + return Unauthorized(); + } + + byte[] rowVersion; + try + { + rowVersion = Convert.FromBase64String(dto.RowVersionBase64); + } + catch (FormatException) + { + return BadRequest("Invalid RowVersionBase64 format."); + } + + var cancelled = await _manualSyncRequestService.CancelRequestAsync( + id, + username, + rowVersion, + ct); + + if (!cancelled) + { + return Conflict(new { message = "The request has already been completed or cancelled, or the data has been modified by another user." }); + } + + return Ok(new { message = "Request cancelled successfully." }); + } + + private static List GetSupportedSyncTypes(DataSync.Configuration.EtlPipelineConfig pipeline) + { + var types = new List(); + if (pipeline.SupportsMassSync) types.Add("mass"); + if (pipeline.SupportsDailySync) types.Add("daily"); + if (pipeline.SupportsHourlySync) types.Add("hourly"); + return types; + } +} diff --git a/NEW/src/JdeScoping.Api/Controllers/PipelineController.cs b/NEW/src/JdeScoping.Api/Controllers/PipelineController.cs new file mode 100644 index 0000000..8da9d7f --- /dev/null +++ b/NEW/src/JdeScoping.Api/Controllers/PipelineController.cs @@ -0,0 +1,224 @@ +using JdeScoping.Api.Contracts.ManualSync; +using JdeScoping.DataSync.Configuration; +using JdeScoping.DataSync.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; + +namespace JdeScoping.Api.Controllers; + +/// +/// API endpoints for pipeline management and hot reload. +/// +[Route("api/pipelines")] +[ApiController] +[Authorize] +public class PipelineController : ApiControllerBase +{ + private readonly IPipelineRegistry _pipelineRegistry; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline registry. + /// The logger. + public PipelineController( + IPipelineRegistry pipelineRegistry, + ILogger logger) + { + _pipelineRegistry = pipelineRegistry; + _logger = logger; + } + + /// + /// Gets all enabled pipelines with their supported sync types. + /// + /// A list of pipeline information view models. + [HttpGet] + [ProducesResponseType(typeof(List), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status401Unauthorized)] + public ActionResult> GetPipelines() + { + var pipelines = _pipelineRegistry.GetEnabledPipelines() + .Select(p => new PipelineInfoViewModel + { + Name = p.Name, + SupportedSyncTypes = GetSupportedSyncTypes(p) + }) + .ToList(); + + return Ok(pipelines); + } + + /// + /// Gets registry metadata including version and last load time. + /// + /// Registry metadata. + [HttpGet("status")] + [ProducesResponseType(typeof(PipelineRegistryStatusViewModel), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status401Unauthorized)] + public ActionResult GetStatus() + { + var allPipelines = _pipelineRegistry.GetAllPipelines(); + var enabledPipelines = _pipelineRegistry.GetEnabledPipelines(); + + return Ok(new PipelineRegistryStatusViewModel + { + Version = _pipelineRegistry.Version, + LastLoadedAt = _pipelineRegistry.LastLoadedAt, + TotalPipelines = allPipelines.Count, + EnabledPipelines = enabledPipelines.Count + }); + } + + /// + /// Reloads all pipeline definitions from disk. + /// Requires Admin role. + /// + /// Cancellation token. + /// The reload result. + [HttpPost("reload")] + [Authorize(Roles = "Admin")] + [ProducesResponseType(typeof(PipelineReloadResultViewModel), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status401Unauthorized)] + [ProducesResponseType(StatusCodes.Status403Forbidden)] + [ProducesResponseType(StatusCodes.Status500InternalServerError)] + public async Task> ReloadPipelines(CancellationToken ct = default) + { + var username = User.Identity?.Name ?? "unknown"; + + _logger.LogInformation("Pipeline reload requested by {User}", username); + + try + { + var result = await _pipelineRegistry.ReloadAsync(ct); + + _logger.LogInformation( + "Pipeline reload completed: Success={Success}, Loaded={Loaded}, Skipped={Skipped}, Version={Version}", + result.Success, + result.PipelinesLoaded, + result.PipelinesSkipped, + result.NewVersion); + + return Ok(new PipelineReloadResultViewModel + { + Success = result.Success, + PipelinesLoaded = result.PipelinesLoaded, + PipelinesSkipped = result.PipelinesSkipped, + PreviousVersion = result.PreviousVersion, + NewVersion = result.NewVersion, + Errors = result.Errors.Select(e => new PipelineLoadErrorViewModel + { + FileName = e.FileName, + PipelineName = e.PipelineName, + ErrorType = e.ErrorType, + Messages = e.Messages + }).ToList() + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Pipeline reload failed"); + return StatusCode(StatusCodes.Status500InternalServerError, new { message = "Pipeline reload failed. See server logs for details." }); + } + } + + private static List GetSupportedSyncTypes(EtlPipelineConfig pipeline) + { + var types = new List(); + if (pipeline.SupportsMassSync) types.Add("mass"); + if (pipeline.SupportsDailySync) types.Add("daily"); + if (pipeline.SupportsHourlySync) types.Add("hourly"); + return types; + } +} + +/// +/// View model for pipeline registry status. +/// +public class PipelineRegistryStatusViewModel +{ + /// + /// Gets or sets the current registry version. + /// + public int Version { get; set; } + + /// + /// Gets or sets the timestamp of the last successful load. + /// + public DateTime? LastLoadedAt { get; set; } + + /// + /// Gets or sets the total number of pipelines. + /// + public int TotalPipelines { get; set; } + + /// + /// Gets or sets the number of enabled pipelines. + /// + public int EnabledPipelines { get; set; } +} + +/// +/// View model for pipeline reload result. +/// +public class PipelineReloadResultViewModel +{ + /// + /// Gets or sets a value indicating whether the reload was successful. + /// + public bool Success { get; set; } + + /// + /// Gets or sets the number of pipelines loaded. + /// + public int PipelinesLoaded { get; set; } + + /// + /// Gets or sets the number of pipelines skipped. + /// + public int PipelinesSkipped { get; set; } + + /// + /// Gets or sets the previous version. + /// + public int PreviousVersion { get; set; } + + /// + /// Gets or sets the new version. + /// + public int NewVersion { get; set; } + + /// + /// Gets or sets the list of errors. + /// + public List Errors { get; set; } = []; +} + +/// +/// View model for pipeline load error. +/// +public class PipelineLoadErrorViewModel +{ + /// + /// Gets or sets the file name. + /// + public string FileName { get; set; } = string.Empty; + + /// + /// Gets or sets the pipeline name. + /// + public string PipelineName { get; set; } = string.Empty; + + /// + /// Gets or sets the error type. + /// + public string ErrorType { get; set; } = string.Empty; + + /// + /// Gets or sets the error messages. + /// + public List Messages { get; set; } = []; +} diff --git a/NEW/src/JdeScoping.Client/Components/DataSync/NewSyncRequestDialog.razor b/NEW/src/JdeScoping.Client/Components/DataSync/NewSyncRequestDialog.razor new file mode 100644 index 0000000..2f58377 --- /dev/null +++ b/NEW/src/JdeScoping.Client/Components/DataSync/NewSyncRequestDialog.razor @@ -0,0 +1,47 @@ +@* + NewSyncRequestDialog.razor - Dialog for creating new manual sync requests. + + Allows users to select a pipeline and sync type to queue a new data sync request. + The sync type dropdown is filtered based on the selected pipeline's supported types. +*@ +@namespace JdeScoping.Client.Components.DataSync +@using JdeScoping.Core.ViewModels + + + + + + + + + + + @if (!string.IsNullOrEmpty(_selectedPipeline) && !string.IsNullOrEmpty(_selectedSyncType)) + { + + @_selectedSyncType sync will be queued for @_selectedPipeline. + + } + + + + + + diff --git a/NEW/src/JdeScoping.Client/Components/DataSync/NewSyncRequestDialog.razor.cs b/NEW/src/JdeScoping.Client/Components/DataSync/NewSyncRequestDialog.razor.cs new file mode 100644 index 0000000..de4cfd7 --- /dev/null +++ b/NEW/src/JdeScoping.Client/Components/DataSync/NewSyncRequestDialog.razor.cs @@ -0,0 +1,114 @@ +using System.Net.Http.Json; +using JdeScoping.Core.ViewModels; +using Microsoft.AspNetCore.Components; +using Radzen; + +namespace JdeScoping.Client.Components.DataSync; + +/// +/// Dialog component for creating new manual sync requests. +/// +public partial class NewSyncRequestDialog : ComponentBase +{ + /// + /// Gets or sets the Radzen dialog service for closing the dialog. + /// + [Inject] + private DialogService DialogService { get; set; } = default!; + + /// + /// Gets or sets the HTTP client for API calls. + /// + [Inject] + private HttpClient HttpClient { get; set; } = default!; + + /// + /// Gets or sets the available pipelines to select from. + /// + [Parameter] + public List Pipelines { get; set; } = new(); + + private string? _selectedPipeline; + private string? _selectedSyncType; + private List _availableSyncTypes = new(); + private bool _isCreating; + + /// + /// Determines if the Create button should be enabled. + /// + private bool CanCreate => !string.IsNullOrEmpty(_selectedPipeline) && + !string.IsNullOrEmpty(_selectedSyncType) && + !_isCreating; + + /// + /// Handles pipeline selection change by filtering available sync types. + /// + private void OnPipelineChanged() + { + // Reset sync type when pipeline changes + _selectedSyncType = null; + + if (string.IsNullOrEmpty(_selectedPipeline)) + { + _availableSyncTypes = new(); + return; + } + + // Find the selected pipeline and get its supported sync types + var pipeline = Pipelines.FirstOrDefault(p => p.Name == _selectedPipeline); + _availableSyncTypes = pipeline?.SupportedSyncTypes ?? new(); + } + + /// + /// Handles the Cancel button click by closing the dialog with null result. + /// + private void OnCancelAsync() + { + DialogService.Close(null); + } + + /// + /// Handles the Create button click by calling the API and closing with the result. + /// + private async Task OnCreateAsync() + { + if (!CanCreate) + { + return; + } + + _isCreating = true; + StateHasChanged(); + + try + { + var createDto = new CreateManualSyncRequestDto + { + PipelineName = _selectedPipeline!, + SyncType = _selectedSyncType! + }; + + var response = await HttpClient.PostAsJsonAsync("api/manual-sync", createDto); + + if (response.IsSuccessStatusCode) + { + var createdRequest = await response.Content.ReadFromJsonAsync(); + DialogService.Close(createdRequest); + } + else + { + // On error, close with null (parent can show notification if needed) + DialogService.Close(null); + } + } + catch + { + // On exception, close with null + DialogService.Close(null); + } + finally + { + _isCreating = false; + } + } +} diff --git a/NEW/src/JdeScoping.Client/Layout/MainLayout.razor b/NEW/src/JdeScoping.Client/Layout/MainLayout.razor index c78b451..55995fa 100644 --- a/NEW/src/JdeScoping.Client/Layout/MainLayout.razor +++ b/NEW/src/JdeScoping.Client/Layout/MainLayout.razor @@ -13,6 +13,7 @@ New Search Search Queue Refresh Status + Data Sync
diff --git a/NEW/src/JdeScoping.Client/Pages/DataSync/DataSyncRequests.razor b/NEW/src/JdeScoping.Client/Pages/DataSync/DataSyncRequests.razor new file mode 100644 index 0000000..1041a2e --- /dev/null +++ b/NEW/src/JdeScoping.Client/Pages/DataSync/DataSyncRequests.razor @@ -0,0 +1,78 @@ +@* + DataSyncRequests.razor - Main page for viewing and managing manual data sync requests. + + Displays a list of manual sync requests with filtering, sorting, and pagination. + Allows users to create new sync requests and cancel pending ones. +*@ +@page "/data-sync/requests" +@attribute [Authorize] +@using JdeScoping.Core.ViewModels +@using JdeScoping.Client.Components.DataSync + +Data Sync Requests - JDE Scoping Tool + + + + Data Sync Requests + + + + + + + + + + + + Show pending only + + + + + +@if (_isLoading) +{ + +} +else if (!string.IsNullOrEmpty(_errorMessage)) +{ + + @_errorMessage + +} +else if (_requests.Count == 0) +{ + + No sync requests found. Click "New Request" to create one. + +} +else +{ + + + + + + + + + + + + + + + + +} diff --git a/NEW/src/JdeScoping.Client/Pages/DataSync/DataSyncRequests.razor.cs b/NEW/src/JdeScoping.Client/Pages/DataSync/DataSyncRequests.razor.cs new file mode 100644 index 0000000..567a89d --- /dev/null +++ b/NEW/src/JdeScoping.Client/Pages/DataSync/DataSyncRequests.razor.cs @@ -0,0 +1,232 @@ +using JdeScoping.Client.Components.DataSync; +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ViewModels; +using Microsoft.AspNetCore.Components; +using Radzen; +using Radzen.Blazor; + +namespace JdeScoping.Client.Pages.DataSync; + +/// +/// Code-behind for the Data Sync Requests page. +/// +public partial class DataSyncRequests : ComponentBase +{ + /// + /// Gets or sets the manual sync API client. + /// + [Inject] + private IManualSyncApiClient ManualSyncApi { get; set; } = default!; + + /// + /// Gets or sets the pipeline API client. + /// + [Inject] + private IPipelineApiClient PipelineApi { get; set; } = default!; + + /// + /// Gets or sets the Radzen dialog service. + /// + [Inject] + private DialogService DialogService { get; set; } = default!; + + /// + /// Gets or sets the Radzen notification service. + /// + [Inject] + private NotificationService NotificationService { get; set; } = default!; + + private List _requests = new(); + private List _pipelines = new(); + private RadzenDataGrid? _grid; + private bool _isLoading = true; + private bool _isReloadingPipelines; + private bool _showPendingOnly = true; + private string? _errorMessage; + + /// + /// Loads initial data when the component is initialized. + /// + protected override async Task OnInitializedAsync() + { + await LoadPipelinesAsync(); + await LoadRequestsAsync(); + } + + /// + /// Loads the available pipelines for creating new requests. + /// + private async Task LoadPipelinesAsync() + { + var result = await ManualSyncApi.GetPipelinesAsync(); + result.Switch( + pipelines => { _pipelines = pipelines.ToList(); }, + _ => { /* Not found - use empty list */ }, + _ => { /* Validation error - use empty list */ }, + _ => { /* Unauthorized - use empty list */ }, + _ => { /* Forbidden - use empty list */ }, + _ => { /* Error - use empty list */ } + ); + } + + /// + /// Loads the manual sync requests from the API. + /// + private async Task LoadRequestsAsync() + { + _isLoading = true; + _errorMessage = null; + StateHasChanged(); + + try + { + var result = await ManualSyncApi.GetRequestsAsync(_showPendingOnly); + result.Switch( + requests => { _requests = requests.ToList(); }, + _ => { _errorMessage = "Requests not found."; _requests = new(); }, + validation => { _errorMessage = string.Join("; ", validation.FieldErrors.SelectMany(e => e.Value)); }, + _ => { _errorMessage = "Session expired. Please login again."; }, + _ => { _errorMessage = "Access denied."; }, + error => { _errorMessage = error.Message; } + ); + } + finally + { + _isLoading = false; + StateHasChanged(); + } + } + + /// + /// Opens the new sync request dialog. + /// + private async Task OpenNewRequestDialogAsync() + { + var result = await DialogService.OpenAsync( + "New Sync Request", + new Dictionary { { "Pipelines", _pipelines } }, + new DialogOptions + { + Width = "400px", + Height = "auto", + CloseDialogOnOverlayClick = false + }); + + if (result is ManualSyncRequestViewModel createdRequest) + { + NotificationService.Notify(NotificationSeverity.Success, "Success", $"Sync request for {createdRequest.PipelineName} created."); + await LoadRequestsAsync(); + } + } + + /// + /// Cancels a pending sync request after user confirmation. + /// + /// The request to cancel. + private async Task CancelRequestAsync(ManualSyncRequestViewModel request) + { + var confirmed = await DialogService.Confirm( + $"Are you sure you want to cancel the {request.SyncType} sync request for {request.PipelineName}?", + "Confirm Cancel", + new ConfirmOptions + { + OkButtonText = "Cancel Request", + CancelButtonText = "Keep Request" + }); + + if (confirmed != true) + { + return; + } + + var result = await ManualSyncApi.CancelRequestAsync(request.Id, request.RowVersionBase64); + var shouldRefresh = false; + result.Switch( + unit => + { + NotificationService.Notify(NotificationSeverity.Success, "Success", "Sync request cancelled."); + shouldRefresh = true; + }, + notFound => { NotificationService.Notify(NotificationSeverity.Warning, "Warning", "Request not found."); }, + validation => { NotificationService.Notify(NotificationSeverity.Error, "Error", string.Join("; ", validation.FieldErrors.SelectMany(e => e.Value))); }, + unauthorized => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Session expired. Please login again."); }, + forbidden => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Access denied."); }, + error => { NotificationService.Notify(NotificationSeverity.Error, "Error", error.Message); } + ); + + if (shouldRefresh) + { + await LoadRequestsAsync(); + } + } + + /// + /// Handles filter checkbox change. + /// + /// The new checkbox value. + private async Task OnFilterChanged(bool value) + { + _showPendingOnly = value; + await LoadRequestsAsync(); + } + + /// + /// Gets the badge style based on the request status. + /// + /// The request status. + /// The appropriate badge style. + private static BadgeStyle GetBadgeStyle(string status) => status switch + { + "Pending" => BadgeStyle.Warning, + "Completed" => BadgeStyle.Success, + "Cancelled" => BadgeStyle.Light, + _ => BadgeStyle.Light + }; + + /// + /// Reloads pipeline definitions from disk (Admin only). + /// + private async Task ReloadPipelinesAsync() + { + _isReloadingPipelines = true; + StateHasChanged(); + + try + { + var result = await PipelineApi.ReloadPipelinesAsync(); + result.Switch( + reloadResult => + { + if (reloadResult.Success) + { + NotificationService.Notify( + NotificationSeverity.Success, + "Pipelines Reloaded", + $"Loaded {reloadResult.PipelinesLoaded} pipelines (version {reloadResult.NewVersion})"); + + // Refresh the pipeline list for the dialog + _ = LoadPipelinesAsync(); + } + else + { + var errorMessages = string.Join("; ", reloadResult.Errors.SelectMany(e => e.Messages)); + NotificationService.Notify( + NotificationSeverity.Error, + "Reload Failed", + $"Pipeline reload failed: {errorMessages}"); + } + }, + _ => { NotificationService.Notify(NotificationSeverity.Warning, "Warning", "Pipeline endpoint not found."); }, + validation => { NotificationService.Notify(NotificationSeverity.Error, "Error", string.Join("; ", validation.FieldErrors.SelectMany(e => e.Value))); }, + _ => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Session expired. Please login again."); }, + _ => { NotificationService.Notify(NotificationSeverity.Error, "Access Denied", "You must be an Admin to reload pipelines."); }, + error => { NotificationService.Notify(NotificationSeverity.Error, "Error", error.Message); } + ); + } + finally + { + _isReloadingPipelines = false; + StateHasChanged(); + } + } +} diff --git a/NEW/src/JdeScoping.Client/Program.cs b/NEW/src/JdeScoping.Client/Program.cs index 054aa36..736a698 100644 --- a/NEW/src/JdeScoping.Client/Program.cs +++ b/NEW/src/JdeScoping.Client/Program.cs @@ -55,6 +55,8 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); // Search services builder.Services.AddScoped(); diff --git a/NEW/src/JdeScoping.Client/Services/ManualSyncApiClient.cs b/NEW/src/JdeScoping.Client/Services/ManualSyncApiClient.cs new file mode 100644 index 0000000..26d1149 --- /dev/null +++ b/NEW/src/JdeScoping.Client/Services/ManualSyncApiClient.cs @@ -0,0 +1,68 @@ +using System.Net.Http.Json; +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of IManualSyncApiClient. +/// +public class ManualSyncApiClient : ApiClientBase, IManualSyncApiClient +{ + /// + /// Initializes a new instance of the class. + /// + /// The HTTP client for API requests. + public ManualSyncApiClient(HttpClient httpClient) : base(httpClient) { } + + /// + /// Gets manual sync requests, optionally filtered to pending only. + /// + /// If true, returns only pending requests. + /// Cancellation token. + /// A list of manual sync request view models. + public Task>> GetRequestsAsync(bool pendingOnly = false, CancellationToken ct = default) + => GetAsync>(ApiRoutes.ManualSync.GetRequests(pendingOnly), ct); + + /// + /// Gets all available pipelines for manual sync. + /// + /// Cancellation token. + /// A list of pipeline information view models. + public Task>> GetPipelinesAsync(CancellationToken ct = default) + => GetAsync>(ApiRoutes.ManualSync.Pipelines, ct); + + /// + /// Creates a new manual sync request. + /// + /// The request details. + /// Cancellation token. + /// The created manual sync request view model. + public Task> CreateRequestAsync(CreateManualSyncRequestDto request, CancellationToken ct = default) + => PostAsync(ApiRoutes.ManualSync.Base, request, ct); + + /// + /// Cancels a pending manual sync request. + /// + /// The request ID. + /// The row version for optimistic concurrency. + /// Cancellation token. + /// A unit result indicating success or failure. + public Task> CancelRequestAsync(int id, string rowVersionBase64, CancellationToken ct = default) + { + var cancelDto = new CancelManualSyncRequestDto { RowVersionBase64 = rowVersionBase64 }; + return PostAsync(ApiRoutes.ManualSync.GetCancel(id), cancelDto, ct); + } +} + +/// +/// DTO for cancelling a manual sync request. +/// +internal class CancelManualSyncRequestDto +{ + /// + /// The row version for optimistic concurrency (Base64 encoded). + /// + public string RowVersionBase64 { get; set; } = string.Empty; +} diff --git a/NEW/src/JdeScoping.Client/Services/PipelineApiClient.cs b/NEW/src/JdeScoping.Client/Services/PipelineApiClient.cs new file mode 100644 index 0000000..7394824 --- /dev/null +++ b/NEW/src/JdeScoping.Client/Services/PipelineApiClient.cs @@ -0,0 +1,29 @@ +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of IPipelineApiClient. +/// +public class PipelineApiClient : ApiClientBase, IPipelineApiClient +{ + /// + /// Initializes a new instance of the class. + /// + /// The HTTP client for API requests. + public PipelineApiClient(HttpClient httpClient) : base(httpClient) { } + + /// + public Task>> GetPipelinesAsync(CancellationToken ct = default) + => GetAsync>(ApiRoutes.Pipeline.Base, ct); + + /// + public Task> GetStatusAsync(CancellationToken ct = default) + => GetAsync(ApiRoutes.Pipeline.Status, ct); + + /// + public Task> ReloadPipelinesAsync(CancellationToken ct = default) + => PostAsync(ApiRoutes.Pipeline.Reload, ct); +} diff --git a/NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs b/NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs index c65f300..4371dd3 100644 --- a/NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs +++ b/NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs @@ -152,4 +152,44 @@ public static class ApiRoutes $"api/refresh-status?minDT={minDt:yyyy-MM-dd}&maxDT={maxDt:yyyy-MM-dd}"; } + /// + /// Routes for manual sync API endpoints. + /// + public static class ManualSync + { + /// Base route for manual sync endpoints. + public const string Base = "api/manual-sync"; + + /// Route for pipeline list. + public const string Pipelines = "api/manual-sync/pipelines"; + + /// Route template for cancelling a request (use in controller attributes). + public const string Cancel = "{id:int}/cancel"; + + /// Builds the route to get requests with optional filter. + /// If true, returns only pending requests. + /// The formatted route. + public static string GetRequests(bool pendingOnly = false) => + pendingOnly ? $"{Base}?pendingOnly=true" : Base; + + /// Builds the route to cancel a specific request. + /// The request ID. + /// The formatted route. + public static string GetCancel(int id) => $"{Base}/{id}/cancel"; + } + + /// + /// Routes for pipeline API endpoints. + /// + public static class Pipeline + { + /// Base route for pipeline endpoints. + public const string Base = "api/pipelines"; + + /// Route for pipeline status. + public const string Status = "api/pipelines/status"; + + /// Route to reload pipelines. + public const string Reload = "api/pipelines/reload"; + } } diff --git a/NEW/src/JdeScoping.Core/ApiContracts/IManualSyncApiClient.cs b/NEW/src/JdeScoping.Core/ApiContracts/IManualSyncApiClient.cs new file mode 100644 index 0000000..cb62812 --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/IManualSyncApiClient.cs @@ -0,0 +1,42 @@ +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for manual sync API operations. +/// +public interface IManualSyncApiClient +{ + /// + /// Gets manual sync requests, optionally filtered to pending only. + /// + /// If true, returns only pending requests. + /// Cancellation token. + /// A list of manual sync request view models. + Task>> GetRequestsAsync(bool pendingOnly = false, CancellationToken ct = default); + + /// + /// Gets all available pipelines for manual sync. + /// + /// Cancellation token. + /// A list of pipeline information view models. + Task>> GetPipelinesAsync(CancellationToken ct = default); + + /// + /// Creates a new manual sync request. + /// + /// The request details. + /// Cancellation token. + /// The created manual sync request view model. + Task> CreateRequestAsync(CreateManualSyncRequestDto request, CancellationToken ct = default); + + /// + /// Cancels a pending manual sync request. + /// + /// The request ID. + /// The row version for optimistic concurrency. + /// Cancellation token. + /// A unit result indicating success or failure. + Task> CancelRequestAsync(int id, string rowVersionBase64, CancellationToken ct = default); +} diff --git a/NEW/src/JdeScoping.Core/ApiContracts/IPipelineApiClient.cs b/NEW/src/JdeScoping.Core/ApiContracts/IPipelineApiClient.cs new file mode 100644 index 0000000..946e9c0 --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/IPipelineApiClient.cs @@ -0,0 +1,120 @@ +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for pipeline API operations. +/// +public interface IPipelineApiClient +{ + /// + /// Gets all enabled pipelines. + /// + /// Cancellation token. + /// A list of pipeline information view models. + Task>> GetPipelinesAsync(CancellationToken ct = default); + + /// + /// Gets the pipeline registry status. + /// + /// Cancellation token. + /// The registry status. + Task> GetStatusAsync(CancellationToken ct = default); + + /// + /// Reloads all pipeline definitions from disk. + /// Requires Admin role. + /// + /// Cancellation token. + /// The reload result. + Task> ReloadPipelinesAsync(CancellationToken ct = default); +} + +/// +/// View model for pipeline registry status. +/// +public class PipelineRegistryStatusViewModel +{ + /// + /// Gets or sets the current registry version. + /// + public int Version { get; set; } + + /// + /// Gets or sets the timestamp of the last successful load. + /// + public DateTime? LastLoadedAt { get; set; } + + /// + /// Gets or sets the total number of pipelines. + /// + public int TotalPipelines { get; set; } + + /// + /// Gets or sets the number of enabled pipelines. + /// + public int EnabledPipelines { get; set; } +} + +/// +/// View model for pipeline reload result. +/// +public class PipelineReloadResultViewModel +{ + /// + /// Gets or sets a value indicating whether the reload was successful. + /// + public bool Success { get; set; } + + /// + /// Gets or sets the number of pipelines loaded. + /// + public int PipelinesLoaded { get; set; } + + /// + /// Gets or sets the number of pipelines skipped. + /// + public int PipelinesSkipped { get; set; } + + /// + /// Gets or sets the previous version. + /// + public int PreviousVersion { get; set; } + + /// + /// Gets or sets the new version. + /// + public int NewVersion { get; set; } + + /// + /// Gets or sets the list of errors. + /// + public List Errors { get; set; } = []; +} + +/// +/// View model for pipeline load error. +/// +public class PipelineLoadErrorViewModel +{ + /// + /// Gets or sets the file name. + /// + public string FileName { get; set; } = string.Empty; + + /// + /// Gets or sets the pipeline name. + /// + public string PipelineName { get; set; } = string.Empty; + + /// + /// Gets or sets the error type. + /// + public string ErrorType { get; set; } = string.Empty; + + /// + /// Gets or sets the error messages. + /// + public List Messages { get; set; } = []; +} diff --git a/NEW/src/JdeScoping.Core/ViewModels/CreateManualSyncRequestDto.cs b/NEW/src/JdeScoping.Core/ViewModels/CreateManualSyncRequestDto.cs new file mode 100644 index 0000000..81f9cfa --- /dev/null +++ b/NEW/src/JdeScoping.Core/ViewModels/CreateManualSyncRequestDto.cs @@ -0,0 +1,17 @@ +namespace JdeScoping.Core.ViewModels; + +/// +/// Request to create a new manual sync request. +/// +public class CreateManualSyncRequestDto +{ + /// + /// The name of the pipeline to sync. + /// + public string PipelineName { get; set; } = string.Empty; + + /// + /// The type of sync to perform (mass, daily, hourly). + /// + public string SyncType { get; set; } = string.Empty; +} diff --git a/NEW/src/JdeScoping.Core/ViewModels/ManualSyncRequestViewModel.cs b/NEW/src/JdeScoping.Core/ViewModels/ManualSyncRequestViewModel.cs new file mode 100644 index 0000000..9b6cedb --- /dev/null +++ b/NEW/src/JdeScoping.Core/ViewModels/ManualSyncRequestViewModel.cs @@ -0,0 +1,57 @@ +namespace JdeScoping.Core.ViewModels; + +/// +/// View model for a manual sync request. +/// +public class ManualSyncRequestViewModel +{ + /// + /// The unique identifier for the sync request. + /// + public int Id { get; set; } + + /// + /// The name of the pipeline being synced. + /// + public string PipelineName { get; set; } = string.Empty; + + /// + /// The type of sync (mass, daily, hourly). + /// + public string SyncType { get; set; } = string.Empty; + + /// + /// The date and time the request was made. + /// + public DateTime RequestDT { get; set; } + + /// + /// The username of the person who requested the sync. + /// + public string RequestedBy { get; set; } = string.Empty; + + /// + /// The date and time the sync completed, if applicable. + /// + public DateTime? CompletedDT { get; set; } + + /// + /// The date and time the sync was cancelled, if applicable. + /// + public DateTime? CancelDT { get; set; } + + /// + /// The username of the person who cancelled the sync, if applicable. + /// + public string? CancelledBy { get; set; } + + /// + /// The current status of the sync request. + /// + public string Status { get; set; } = string.Empty; + + /// + /// The row version for optimistic concurrency (Base64 encoded). + /// + public string RowVersionBase64 { get; set; } = string.Empty; +} diff --git a/NEW/src/JdeScoping.Core/ViewModels/PipelineInfoViewModel.cs b/NEW/src/JdeScoping.Core/ViewModels/PipelineInfoViewModel.cs new file mode 100644 index 0000000..e8d18dd --- /dev/null +++ b/NEW/src/JdeScoping.Core/ViewModels/PipelineInfoViewModel.cs @@ -0,0 +1,17 @@ +namespace JdeScoping.Core.ViewModels; + +/// +/// View model for pipeline information. +/// +public class PipelineInfoViewModel +{ + /// + /// The name of the pipeline. + /// + public string Name { get; set; } = string.Empty; + + /// + /// The sync types supported by this pipeline. + /// + public List SupportedSyncTypes { get; set; } = new(); +} diff --git a/NEW/src/JdeScoping.DataAccess/DependencyInjection.cs b/NEW/src/JdeScoping.DataAccess/DependencyInjection.cs index a73860a..428b75e 100644 --- a/NEW/src/JdeScoping.DataAccess/DependencyInjection.cs +++ b/NEW/src/JdeScoping.DataAccess/DependencyInjection.cs @@ -52,6 +52,9 @@ public static class DataAccessDependencyInjection services.AddScoped(); services.AddScoped(); + // Register manual sync request service (scoped - per request lifetime) + services.AddScoped(); + return services; } diff --git a/NEW/src/JdeScoping.DataAccess/JdeScoping.DataAccess.csproj b/NEW/src/JdeScoping.DataAccess/JdeScoping.DataAccess.csproj index 1d8ac5e..f13cff9 100644 --- a/NEW/src/JdeScoping.DataAccess/JdeScoping.DataAccess.csproj +++ b/NEW/src/JdeScoping.DataAccess/JdeScoping.DataAccess.csproj @@ -21,6 +21,7 @@ + diff --git a/NEW/src/JdeScoping.DataAccess/Services/IManualSyncRequestService.cs b/NEW/src/JdeScoping.DataAccess/Services/IManualSyncRequestService.cs new file mode 100644 index 0000000..4c8b487 --- /dev/null +++ b/NEW/src/JdeScoping.DataAccess/Services/IManualSyncRequestService.cs @@ -0,0 +1,68 @@ +using JdeScoping.Domain.Models; + +namespace JdeScoping.DataAccess.Services; + +/// +/// Service for managing manual data sync requests. +/// +public interface IManualSyncRequestService +{ + /// + /// Gets all manual sync requests, optionally filtered to pending only. + /// + /// If true, returns only pending requests. + /// Cancellation token. + /// A read-only list of manual sync requests. + Task> GetRequestsAsync( + bool pendingOnly = false, + CancellationToken ct = default); + + /// + /// Gets the next pending request in FIFO order (for processor). + /// + /// Cancellation token. + /// The next pending request, or null if none exists. + Task GetNextPendingRequestAsync(CancellationToken ct = default); + + /// + /// Creates a new manual sync request. + /// + /// The name of the ETL pipeline to sync. + /// The type of sync (mass, daily, hourly). + /// The username of the requester. + /// Cancellation token. + /// The created manual sync request. + Task CreateRequestAsync( + string pipelineName, + string syncType, + string requestedBy, + CancellationToken ct = default); + + /// + /// Cancels a pending request using optimistic concurrency. + /// Returns true if cancelled, false if already completed/cancelled. + /// + /// The request ID. + /// The username of the user cancelling the request. + /// The row version for optimistic concurrency. + /// Cancellation token. + /// True if the request was cancelled; false if already completed or cancelled. + Task CancelRequestAsync( + int id, + string cancelledBy, + byte[] rowVersion, + CancellationToken ct = default); + + /// + /// Marks a request as completed using optimistic concurrency. + /// Called by the sync processor. + /// + /// The request ID. + /// The row version for optimistic concurrency. + /// Cancellation token. + /// True if the request was marked completed; false if already completed or cancelled. + Task CompleteRequestAsync( + int id, + byte[] rowVersion, + CancellationToken ct = default); +} diff --git a/NEW/src/JdeScoping.DataAccess/Services/ManualSyncRequestService.cs b/NEW/src/JdeScoping.DataAccess/Services/ManualSyncRequestService.cs new file mode 100644 index 0000000..854b674 --- /dev/null +++ b/NEW/src/JdeScoping.DataAccess/Services/ManualSyncRequestService.cs @@ -0,0 +1,202 @@ +using Dapper; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.Domain.Models; +using Microsoft.Extensions.Logging; + +namespace JdeScoping.DataAccess.Services; + +/// +/// Service implementation for managing manual data sync requests. +/// +public sealed class ManualSyncRequestService : IManualSyncRequestService +{ + private readonly IDbConnectionFactory _connectionFactory; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The database connection factory. + /// The logger instance. + public ManualSyncRequestService( + IDbConnectionFactory connectionFactory, + ILogger logger) + { + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public async Task> GetRequestsAsync( + bool pendingOnly = false, + CancellationToken ct = default) + { + _logger.LogDebug("Getting manual sync requests (pendingOnly: {PendingOnly})", pendingOnly); + + const string sqlAll = """ + SELECT ID, PipelineName, SyncType, RequestDT, RequestedBy, + CompletedDT, CancelDT, CancelledBy, RowVersion + FROM dbo.ManualSyncRequest + ORDER BY RequestDT DESC + """; + + const string sqlPending = """ + SELECT ID, PipelineName, SyncType, RequestDT, RequestedBy, + CompletedDT, CancelDT, CancelledBy, RowVersion + FROM dbo.ManualSyncRequest + WHERE CompletedDT IS NULL AND CancelDT IS NULL + ORDER BY RequestDT ASC + """; + + await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct); + var results = await connection.QueryAsync( + pendingOnly ? sqlPending : sqlAll); + + var list = results.ToList(); + _logger.LogDebug("Retrieved {Count} manual sync requests", list.Count); + return list; + } + + /// + public async Task GetNextPendingRequestAsync(CancellationToken ct = default) + { + _logger.LogDebug("Getting next pending manual sync request"); + + const string sql = """ + SELECT TOP 1 ID, PipelineName, SyncType, RequestDT, RequestedBy, + CompletedDT, CancelDT, CancelledBy, RowVersion + FROM dbo.ManualSyncRequest + WHERE CompletedDT IS NULL AND CancelDT IS NULL + ORDER BY RequestDT ASC + """; + + await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct); + var result = await connection.QueryFirstOrDefaultAsync(sql); + + if (result != null) + { + _logger.LogDebug("Found pending request ID {Id} for pipeline {Pipeline}", + result.Id, result.PipelineName); + } + else + { + _logger.LogDebug("No pending manual sync requests found"); + } + + return result; + } + + /// + public async Task CreateRequestAsync( + string pipelineName, + string syncType, + string requestedBy, + CancellationToken ct = default) + { + _logger.LogInformation( + "Creating manual sync request for pipeline {Pipeline}, type {SyncType}, by {User}", + pipelineName, syncType, requestedBy); + + const string sql = """ + INSERT INTO dbo.ManualSyncRequest (PipelineName, SyncType, RequestedBy) + OUTPUT INSERTED.ID, INSERTED.PipelineName, INSERTED.SyncType, + INSERTED.RequestDT, INSERTED.RequestedBy, + INSERTED.CompletedDT, INSERTED.CancelDT, INSERTED.CancelledBy, + INSERTED.RowVersion + VALUES (@PipelineName, @SyncType, @RequestedBy) + """; + + await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct); + var result = await connection.QuerySingleAsync(sql, new + { + PipelineName = pipelineName, + SyncType = syncType, + RequestedBy = requestedBy + }); + + _logger.LogInformation("Created manual sync request with ID {Id}", result.Id); + return result; + } + + /// + public async Task CancelRequestAsync( + int id, + string cancelledBy, + byte[] rowVersion, + CancellationToken ct = default) + { + _logger.LogInformation( + "Cancelling manual sync request ID {Id} by {User}", + id, cancelledBy); + + const string sql = """ + UPDATE dbo.ManualSyncRequest + SET CancelDT = @CancelDT, CancelledBy = @CancelledBy + WHERE ID = @Id AND RowVersion = @RowVersion + AND CompletedDT IS NULL AND CancelDT IS NULL + """; + + await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct); + var affectedRows = await connection.ExecuteAsync(sql, new + { + Id = id, + CancelDT = DateTime.UtcNow, + CancelledBy = cancelledBy, + RowVersion = rowVersion + }); + + var success = affectedRows > 0; + + if (success) + { + _logger.LogInformation("Successfully cancelled manual sync request ID {Id}", id); + } + else + { + _logger.LogWarning( + "Failed to cancel manual sync request ID {Id} - already completed/cancelled or version mismatch", + id); + } + + return success; + } + + /// + public async Task CompleteRequestAsync( + int id, + byte[] rowVersion, + CancellationToken ct = default) + { + _logger.LogInformation("Completing manual sync request ID {Id}", id); + + const string sql = """ + UPDATE dbo.ManualSyncRequest + SET CompletedDT = @CompletedDT + WHERE ID = @Id AND RowVersion = @RowVersion + AND CompletedDT IS NULL AND CancelDT IS NULL + """; + + await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct); + var affectedRows = await connection.ExecuteAsync(sql, new + { + Id = id, + CompletedDT = DateTime.UtcNow, + RowVersion = rowVersion + }); + + var success = affectedRows > 0; + + if (success) + { + _logger.LogInformation("Successfully completed manual sync request ID {Id}", id); + } + else + { + _logger.LogWarning( + "Failed to complete manual sync request ID {Id} - already completed/cancelled or version mismatch", + id); + } + + return success; + } +} diff --git a/NEW/src/JdeScoping.DataSync/Configuration/DestinationElement.cs b/NEW/src/JdeScoping.DataSync/Configuration/DestinationElement.cs new file mode 100644 index 0000000..a1ee354 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Configuration/DestinationElement.cs @@ -0,0 +1,22 @@ +namespace JdeScoping.DataSync.Configuration; + +/// +/// Configuration for the pipeline destination. +/// +public class DestinationElement +{ + /// + /// Target table name in the cache database. + /// + public string Table { get; set; } = string.Empty; + + /// + /// Columns used to match existing records for upsert. + /// + public List MatchColumns { get; set; } = []; + + /// + /// Columns to exclude from UPDATE operations. + /// + public List ExcludeFromUpdate { get; set; } = []; +} diff --git a/NEW/src/JdeScoping.DataSync/Configuration/EtlPipelineConfig.cs b/NEW/src/JdeScoping.DataSync/Configuration/EtlPipelineConfig.cs new file mode 100644 index 0000000..92722e5 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Configuration/EtlPipelineConfig.cs @@ -0,0 +1,78 @@ +namespace JdeScoping.DataSync.Configuration; + +/// +/// Represents an ETL pipeline definition loaded from a JSON file. +/// +public class EtlPipelineConfig +{ + /// + /// Unique name of the pipeline (must match filename, case-insensitive). + /// + public string Name { get; set; } = string.Empty; + + /// + /// Whether the pipeline is enabled for execution. + /// + public bool IsEnabled { get; set; } = true; + + /// + /// If true, pipeline can only be triggered via ManualSyncRequest. + /// No interval validation is required for manual-only pipelines. + /// + public bool IsManualOnly { get; set; } + + /// + /// Interval for mass sync in minutes. Null if mass sync not supported. + /// + public int? MassSyncIntervalMinutes { get; set; } + + /// + /// Interval for daily sync in minutes. Null if daily sync not supported. + /// + public int? DailySyncIntervalMinutes { get; set; } + + /// + /// Interval for hourly sync in minutes. Null if hourly sync not supported. + /// + public int? HourlySyncIntervalMinutes { get; set; } + + /// + /// Scripts to run before the main sync. Optional. + /// + public List PreScripts { get; set; } = []; + + /// + /// The data source configuration. Required. + /// + public SourceElement Source { get; set; } = null!; + + /// + /// Data transformations to apply. Optional. + /// + public List Transforms { get; set; } = []; + + /// + /// The destination configuration. Required. + /// + public DestinationElement Destination { get; set; } = null!; + + /// + /// Scripts to run after the main sync. Optional. + /// + public List PostScripts { get; set; } = []; + + /// + /// Gets a value indicating whether the pipeline supports mass sync. + /// + public bool SupportsMassSync => MassSyncIntervalMinutes.HasValue; + + /// + /// Gets a value indicating whether the pipeline supports daily sync. + /// + public bool SupportsDailySync => DailySyncIntervalMinutes.HasValue; + + /// + /// Gets a value indicating whether the pipeline supports hourly sync. + /// + public bool SupportsHourlySync => HourlySyncIntervalMinutes.HasValue; +} diff --git a/NEW/src/JdeScoping.DataSync/Configuration/ParameterElement.cs b/NEW/src/JdeScoping.DataSync/Configuration/ParameterElement.cs new file mode 100644 index 0000000..5d2eadb --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Configuration/ParameterElement.cs @@ -0,0 +1,27 @@ +namespace JdeScoping.DataSync.Configuration; + +/// +/// Configuration for a query parameter. +/// +public class ParameterElement +{ + /// + /// Parameter name as used in query (e.g., ":dateUpdated"). + /// + public string Name { get; set; } = string.Empty; + + /// + /// Format conversion (jdeJulian, jdeTime, etc.). + /// + public string? Format { get; set; } + + /// + /// Source of the value (offset = from last sync time). + /// + public string Source { get; set; } = "offset"; + + /// + /// Static value if source is not offset. + /// + public string? Value { get; set; } +} diff --git a/NEW/src/JdeScoping.DataSync/Configuration/ScriptElement.cs b/NEW/src/JdeScoping.DataSync/Configuration/ScriptElement.cs new file mode 100644 index 0000000..157a32a --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Configuration/ScriptElement.cs @@ -0,0 +1,17 @@ +namespace JdeScoping.DataSync.Configuration; + +/// +/// Configuration for a pre/post script. +/// +public class ScriptElement +{ + /// + /// Connection identifier for script execution. + /// + public string Connection { get; set; } = "lotfinder"; + + /// + /// SQL script to execute. + /// + public string Script { get; set; } = string.Empty; +} diff --git a/NEW/src/JdeScoping.DataSync/Configuration/SourceElement.cs b/NEW/src/JdeScoping.DataSync/Configuration/SourceElement.cs new file mode 100644 index 0000000..b601476 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Configuration/SourceElement.cs @@ -0,0 +1,27 @@ +namespace JdeScoping.DataSync.Configuration; + +/// +/// Configuration for the pipeline data source. +/// +public class SourceElement +{ + /// + /// Connection identifier (jde, cms, giw, lotfinder). + /// + public string Connection { get; set; } = string.Empty; + + /// + /// Query for incremental syncs (daily/hourly). + /// + public string Query { get; set; } = string.Empty; + + /// + /// Query for mass sync. Falls back to Query if not specified. + /// + public string? MassQuery { get; set; } + + /// + /// Query parameters with format and source configuration. + /// + public Dictionary Parameters { get; set; } = new(); +} diff --git a/NEW/src/JdeScoping.DataSync/Configuration/TransformElement.cs b/NEW/src/JdeScoping.DataSync/Configuration/TransformElement.cs new file mode 100644 index 0000000..095ba29 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Configuration/TransformElement.cs @@ -0,0 +1,21 @@ +using System.Text.Json; + +namespace JdeScoping.DataSync.Configuration; + +/// +/// Configuration for a data transformation. +/// +public class TransformElement +{ + /// + /// Type of transformation (ColumnDrop, ColumnRename, JdeDate, Regex, etc.). + /// + public string TransformType { get; set; } = string.Empty; + + /// + /// Transform-specific configuration as raw JSON. + /// Using JsonElement avoids Dictionary<string, object> deserialization issues + /// where values would become JsonElement anyway without custom converters. + /// + public JsonElement? Config { get; set; } +} diff --git a/NEW/src/JdeScoping.DataSync/Contracts/ISyncOrchestrator.cs b/NEW/src/JdeScoping.DataSync/Contracts/ISyncOrchestrator.cs index 672c0d9..d1d30fd 100644 --- a/NEW/src/JdeScoping.DataSync/Contracts/ISyncOrchestrator.cs +++ b/NEW/src/JdeScoping.DataSync/Contracts/ISyncOrchestrator.cs @@ -1,3 +1,5 @@ +using JdeScoping.DataSync.Models; + namespace JdeScoping.DataSync.Contracts; /// @@ -11,4 +13,12 @@ public interface ISyncOrchestrator /// Cancellation token for graceful shutdown. /// A task representing the async operation. Task ExecutePendingSyncsAsync(CancellationToken cancellationToken = default); + + /// + /// Executes a single sync operation (used for manual sync requests). + /// + /// The sync task to execute. + /// Cancellation token for graceful shutdown. + /// A task representing the async operation. + Task ExecuteSingleSyncAsync(DataUpdateTask task, CancellationToken cancellationToken = default); } diff --git a/NEW/src/JdeScoping.DataSync/DependencyInjection.cs b/NEW/src/JdeScoping.DataSync/DependencyInjection.cs index 8a64d29..cb6ff69 100644 --- a/NEW/src/JdeScoping.DataSync/DependencyInjection.cs +++ b/NEW/src/JdeScoping.DataSync/DependencyInjection.cs @@ -42,6 +42,13 @@ public static class DataSyncDependencyInjection // Pipeline factory (new ETL infrastructure) services.AddSingleton(); + // Pipeline registry services (new hot-reload infrastructure) + services.AddSingleton(); + services.AddSingleton(); + + // Pipeline registry initializer - runs before WorkProcessor to ensure pipelines load first + services.AddHostedService(); + // Register hosted service (WorkProcessor combines data sync and search processing) services.AddHostedService(); diff --git a/NEW/src/JdeScoping.DataSync/Models/DataUpdateTask.cs b/NEW/src/JdeScoping.DataSync/Models/DataUpdateTask.cs index ac854f7..0ffadfc 100644 --- a/NEW/src/JdeScoping.DataSync/Models/DataUpdateTask.cs +++ b/NEW/src/JdeScoping.DataSync/Models/DataUpdateTask.cs @@ -1,4 +1,5 @@ using JdeScoping.Core.Models.Enums; +using JdeScoping.DataSync.Configuration; using JdeScoping.DataSync.Options; namespace JdeScoping.DataSync.Models; @@ -19,7 +20,7 @@ public class DataUpdateTask public required string TableName { get; init; } /// - /// Source system: "JDE" or "CMS". + /// Source system: "JDE", "CMS", "GIW", or "LOTFINDER". /// public required string SourceSystem { get; init; } @@ -40,9 +41,15 @@ public class DataUpdateTask public DateTime? MinimumDt { get; init; } /// - /// The data source configuration for this task. + /// The pipeline configuration for this task (new format). /// - public required DataSourceConfig Config { get; init; } + public EtlPipelineConfig? Pipeline { get; init; } + + /// + /// The data source configuration for this task (legacy format - will be removed). + /// + [Obsolete("Use Pipeline instead. This property exists for backward compatibility during migration.")] + public DataSourceConfig? Config { get; init; } /// /// Gets a unique key for logging purposes. @@ -50,13 +57,47 @@ public class DataUpdateTask public string LogKey => $"{TableName}_{UpdateType}_{OperationId:N}"; /// - /// Gets the schedule configuration for this update type. + /// Gets the interval in minutes for this update type from the pipeline config. /// - public ScheduleConfig ScheduleConfig => UpdateType switch + public int? IntervalMinutes => Pipeline != null ? UpdateType switch { - UpdateTypes.Mass => Config.MassConfig, - UpdateTypes.Daily => Config.DailyConfig, - UpdateTypes.Hourly => Config.HourlyConfig, - _ => throw new ArgumentOutOfRangeException(nameof(UpdateType)) - }; + UpdateTypes.Mass => Pipeline.MassSyncIntervalMinutes, + UpdateTypes.Daily => Pipeline.DailySyncIntervalMinutes, + UpdateTypes.Hourly => Pipeline.HourlySyncIntervalMinutes, + _ => null + } : null; + + /// + /// Creates a DataUpdateTask from an EtlPipelineConfig. + /// + public static DataUpdateTask FromPipeline( + EtlPipelineConfig pipeline, + UpdateTypes updateType, + DateTime? minimumDt = null) + { + return new DataUpdateTask + { + TableName = pipeline.Destination.Table, + SourceSystem = MapConnectionToSourceSystem(pipeline.Source.Connection), + SourceData = pipeline.Name, + UpdateType = updateType, + MinimumDt = minimumDt, + Pipeline = pipeline + }; + } + + /// + /// Maps connection identifier to source system name. + /// + private static string MapConnectionToSourceSystem(string connection) + { + return connection.ToUpperInvariant() switch + { + "JDE" => "JDE", + "CMS" => "CMS", + "GIW" => "GIW", + "LOTFINDER" => "LOTFINDER", + _ => connection.ToUpperInvariant() + }; + } } diff --git a/NEW/src/JdeScoping.DataSync/Options/DataSyncOptions.cs b/NEW/src/JdeScoping.DataSync/Options/DataSyncOptions.cs index 45dee4f..efc6c50 100644 --- a/NEW/src/JdeScoping.DataSync/Options/DataSyncOptions.cs +++ b/NEW/src/JdeScoping.DataSync/Options/DataSyncOptions.cs @@ -58,6 +58,18 @@ public class DataSyncOptions /// public bool Enabled { get; set; } = true; + /// + /// Directory containing pipeline.*.json files. + /// Resolved relative to content root. + /// + public string PipelinesDirectory { get; set; } = "Pipelines"; + + /// + /// If true (default), startup fails if any enabled pipeline is invalid. + /// If false, invalid enabled pipelines are skipped with warnings. + /// + public bool StrictPipelineValidation { get; set; } = true; + /// /// Per-table data source configurations. /// diff --git a/NEW/src/JdeScoping.DataSync/Options/WorkProcessorOptions.cs b/NEW/src/JdeScoping.DataSync/Options/WorkProcessorOptions.cs index 9db4eb0..afc699d 100644 --- a/NEW/src/JdeScoping.DataSync/Options/WorkProcessorOptions.cs +++ b/NEW/src/JdeScoping.DataSync/Options/WorkProcessorOptions.cs @@ -34,4 +34,12 @@ public class WorkProcessorOptions /// [Range(1, 365)] public int PurgeRetentionDays { get; set; } = 30; + + /// + /// Maximum manual sync requests to process per work cycle. + /// Prevents manual requests from starving scheduled syncs. + /// Default: 5. Set to 0 for unlimited (not recommended). + /// + [Range(0, 100)] + public int MaxManualRequestsPerCycle { get; set; } = 5; } diff --git a/NEW/src/JdeScoping.DataSync/Services/IPipelineRegistry.cs b/NEW/src/JdeScoping.DataSync/Services/IPipelineRegistry.cs new file mode 100644 index 0000000..494fdbc --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Services/IPipelineRegistry.cs @@ -0,0 +1,115 @@ +using JdeScoping.DataSync.Configuration; + +namespace JdeScoping.DataSync.Services; + +/// +/// Registry for ETL pipeline definitions with hot reload support. +/// +public interface IPipelineRegistry +{ + /// + /// Gets all loaded pipelines (immutable snapshot). + /// + IReadOnlyList GetAllPipelines(); + + /// + /// Gets all enabled pipelines (immutable snapshot). + /// + IReadOnlyList GetEnabledPipelines(); + + /// + /// Gets a pipeline by name (case-insensitive). + /// + /// The pipeline name to find. + /// The pipeline if found, or null. + EtlPipelineConfig? GetPipeline(string name); + + /// + /// Validates that a pipeline and sync type combination is valid. + /// + /// The pipeline name. + /// The sync type (mass, daily, hourly). + /// True if the combination is valid. + bool IsValidPipelineAndSyncType(string pipelineName, string syncType); + + /// + /// Reloads all pipelines from disk. + /// Returns validation results for each file. + /// Reload is atomic: on any enabled pipeline failure, keeps previous snapshot. + /// + /// Cancellation token. + /// The reload result. + Task ReloadAsync(CancellationToken ct = default); + + /// + /// Gets the current registry version (increments on successful reload). + /// + int Version { get; } + + /// + /// Gets the timestamp of the last successful load. + /// + DateTime? LastLoadedAt { get; } +} + +/// +/// Result of a pipeline reload operation. +/// +public class PipelineReloadResult +{ + /// + /// Gets or sets a value indicating whether the reload was successful. + /// + public bool Success { get; set; } + + /// + /// Gets or sets the number of pipelines loaded. + /// + public int PipelinesLoaded { get; set; } + + /// + /// Gets or sets the number of pipelines skipped due to errors. + /// + public int PipelinesSkipped { get; set; } + + /// + /// Gets or sets the previous version before reload. + /// + public int PreviousVersion { get; set; } + + /// + /// Gets or sets the new version after reload (unchanged if reload failed). + /// + public int NewVersion { get; set; } + + /// + /// Gets or sets the list of validation errors. + /// + public List Errors { get; set; } = []; +} + +/// +/// Represents an error that occurred while loading a pipeline. +/// +public class PipelineLoadError +{ + /// + /// Gets or sets the file name that had the error. + /// + public string FileName { get; set; } = string.Empty; + + /// + /// Gets or sets the pipeline name (if parseable). + /// + public string PipelineName { get; set; } = string.Empty; + + /// + /// Gets or sets the error type (parse, validation, file). + /// + public string ErrorType { get; set; } = string.Empty; + + /// + /// Gets or sets the error messages. + /// + public List Messages { get; set; } = []; +} diff --git a/NEW/src/JdeScoping.DataSync/Services/IPipelineValidator.cs b/NEW/src/JdeScoping.DataSync/Services/IPipelineValidator.cs new file mode 100644 index 0000000..0c3f908 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Services/IPipelineValidator.cs @@ -0,0 +1,38 @@ +using JdeScoping.DataSync.Configuration; + +namespace JdeScoping.DataSync.Services; + +/// +/// Validates pipeline definitions. +/// +public interface IPipelineValidator +{ + /// + /// Validates a pipeline definition. + /// + /// The pipeline to validate. + /// The source file name for error messages. + /// Validation result with errors and warnings. + PipelineValidationResult Validate(EtlPipelineConfig pipeline, string fileName); +} + +/// +/// Result of validating a single pipeline. +/// +public class PipelineValidationResult +{ + /// + /// Gets or sets a value indicating whether the pipeline is valid. + /// + public bool IsValid { get; set; } + + /// + /// Gets or sets the list of validation errors. + /// + public List Errors { get; set; } = []; + + /// + /// Gets or sets the list of validation warnings. + /// + public List Warnings { get; set; } = []; +} diff --git a/NEW/src/JdeScoping.DataSync/Services/PipelineRegistry.cs b/NEW/src/JdeScoping.DataSync/Services/PipelineRegistry.cs new file mode 100644 index 0000000..2bca478 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Services/PipelineRegistry.cs @@ -0,0 +1,263 @@ +using System.Text.Json; +using JdeScoping.DataSync.Configuration; +using JdeScoping.DataSync.Options; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace JdeScoping.DataSync.Services; + +/// +/// Registry for ETL pipeline definitions with hot reload support. +/// Uses immutable snapshots for thread-safe reads. +/// +public class PipelineRegistry : IPipelineRegistry +{ + private readonly IOptions _options; + private readonly IPipelineValidator _validator; + private readonly ILogger _logger; + private readonly IHostEnvironment _environment; + + /// + /// Immutable snapshot - swapped atomically. + /// + private volatile PipelineSnapshot _snapshot = PipelineSnapshot.Empty; + + /// + /// Serializes reload operations (only one reload at a time). + /// + private readonly SemaphoreSlim _reloadLock = new(1, 1); + + /// + /// Version incremented via Interlocked. + /// + private int _version; + + private static readonly JsonSerializerOptions PipelineJsonOptions = new() + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + /// + /// Initializes a new instance of the class. + /// + public PipelineRegistry( + IOptions options, + IPipelineValidator validator, + ILogger logger, + IHostEnvironment environment) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _validator = validator ?? throw new ArgumentNullException(nameof(validator)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _environment = environment ?? throw new ArgumentNullException(nameof(environment)); + } + + /// + public IReadOnlyList GetAllPipelines() => _snapshot.AllPipelines; + + /// + public IReadOnlyList GetEnabledPipelines() => _snapshot.EnabledPipelines; + + /// + public EtlPipelineConfig? GetPipeline(string name) => _snapshot.GetByName(name); + + /// + public int Version => _version; + + /// + public DateTime? LastLoadedAt => _snapshot.LoadedAt == default ? null : _snapshot.LoadedAt; + + /// + public bool IsValidPipelineAndSyncType(string pipelineName, string syncType) + { + var pipeline = GetPipeline(pipelineName); + if (pipeline == null || !pipeline.IsEnabled) + { + return false; + } + + return syncType.ToLowerInvariant() switch + { + "mass" => pipeline.SupportsMassSync, + "daily" => pipeline.SupportsDailySync, + "hourly" => pipeline.SupportsHourlySync, + _ => false + }; + } + + /// + public async Task ReloadAsync(CancellationToken ct = default) + { + await _reloadLock.WaitAsync(ct); + try + { + var newSnapshot = await LoadSnapshotAsync(ct); + var previousVersion = _version; + + // Atomic swap only on success + if (newSnapshot.Result.Success) + { + Interlocked.Exchange(ref _snapshot, newSnapshot); + Interlocked.Increment(ref _version); + } + + newSnapshot.Result.PreviousVersion = previousVersion; + newSnapshot.Result.NewVersion = _version; + return newSnapshot.Result; + } + finally + { + _reloadLock.Release(); + } + } + + private async Task LoadSnapshotAsync(CancellationToken ct) + { + var result = new PipelineReloadResult(); + var pipelines = new List(); + var seenNames = new HashSet(StringComparer.OrdinalIgnoreCase); + + // Resolve directory path relative to content root + var directory = Path.Combine( + _environment.ContentRootPath, + _options.Value.PipelinesDirectory); + + // Handle missing directory + if (!Directory.Exists(directory)) + { + result.Errors.Add(new PipelineLoadError + { + FileName = directory, + ErrorType = "file", + Messages = ["Pipeline directory does not exist"] + }); + result.Success = false; + return new PipelineSnapshot([], result); + } + + // Load each pipeline file + var files = Directory.GetFiles(directory, "pipeline.*.json"); + _logger.LogDebug("Found {Count} pipeline files in {Directory}", files.Length, directory); + + foreach (var file in files) + { + ct.ThrowIfCancellationRequested(); + + var fileName = Path.GetFileName(file); + EtlPipelineConfig? pipeline = null; + + // Parse JSON + try + { + var json = await File.ReadAllTextAsync(file, ct); + pipeline = JsonSerializer.Deserialize(json, PipelineJsonOptions); + } + catch (JsonException ex) + { + result.Errors.Add(new PipelineLoadError + { + FileName = fileName, + ErrorType = "parse", + Messages = [$"JSON parse error: {ex.Message}"] + }); + result.PipelinesSkipped++; + continue; + } + catch (IOException ex) + { + result.Errors.Add(new PipelineLoadError + { + FileName = fileName, + ErrorType = "file", + Messages = [$"File read error: {ex.Message}"] + }); + result.PipelinesSkipped++; + continue; + } + + if (pipeline == null) + { + result.Errors.Add(new PipelineLoadError + { + FileName = fileName, + ErrorType = "parse", + Messages = ["Deserialized to null"] + }); + result.PipelinesSkipped++; + continue; + } + + // Check for duplicate names + if (!seenNames.Add(pipeline.Name)) + { + result.Errors.Add(new PipelineLoadError + { + FileName = fileName, + PipelineName = pipeline.Name, + ErrorType = "validation", + Messages = [$"Duplicate pipeline name: {pipeline.Name}"] + }); + result.PipelinesSkipped++; + continue; + } + + // Validate pipeline + var validation = _validator.Validate(pipeline, fileName); + if (!validation.IsValid) + { + result.Errors.Add(new PipelineLoadError + { + FileName = fileName, + PipelineName = pipeline.Name, + ErrorType = "validation", + Messages = validation.Errors + }); + + // Only skip if enabled (disabled invalid pipelines are warnings) + if (pipeline.IsEnabled) + { + result.PipelinesSkipped++; + continue; + } + + // Disabled invalid pipelines are logged as warnings but added + _logger.LogWarning( + "Disabled pipeline {Name} has validation errors: {Errors}", + pipeline.Name, + string.Join("; ", validation.Errors)); + } + + // Log warnings + foreach (var warning in validation.Warnings) + { + _logger.LogWarning("Pipeline {Name}: {Warning}", pipeline.Name, warning); + } + + pipelines.Add(pipeline); + result.PipelinesLoaded++; + } + + // Determine success: all enabled pipelines must be valid + // Check if any errors are for enabled pipelines that were skipped + var hasEnabledErrors = result.Errors.Any(e => + { + // If this error caused a pipeline to be skipped (not in our list), + // and it was for an enabled pipeline, it's a failure + var matchingPipeline = pipelines.FirstOrDefault(p => + string.Equals(p.Name, e.PipelineName, StringComparison.OrdinalIgnoreCase)); + + // If pipeline wasn't added to list, it was skipped (enabled + invalid) + return matchingPipeline == null && !string.IsNullOrEmpty(e.PipelineName); + }); + + result.Success = !hasEnabledErrors && result.Errors.All(e => e.ErrorType != "file" || e.Messages.All(m => !m.Contains("does not exist"))); + + return new PipelineSnapshot(pipelines, result); + } +} diff --git a/NEW/src/JdeScoping.DataSync/Services/PipelineRegistryInitializer.cs b/NEW/src/JdeScoping.DataSync/Services/PipelineRegistryInitializer.cs new file mode 100644 index 0000000..e1ee16e --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Services/PipelineRegistryInitializer.cs @@ -0,0 +1,89 @@ +using JdeScoping.DataSync.Options; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace JdeScoping.DataSync.Services; + +/// +/// Initializes the pipeline registry at application startup. +/// Runs as a hosted service to properly handle async loading. +/// +public class PipelineRegistryInitializer : IHostedService +{ + private readonly IPipelineRegistry _registry; + private readonly IOptions _options; + private readonly IHostApplicationLifetime _lifetime; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + public PipelineRegistryInitializer( + IPipelineRegistry registry, + IOptions options, + IHostApplicationLifetime lifetime, + ILogger logger) + { + _registry = registry ?? throw new ArgumentNullException(nameof(registry)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _lifetime = lifetime ?? throw new ArgumentNullException(nameof(lifetime)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public async Task StartAsync(CancellationToken ct) + { + _logger.LogInformation("Loading pipeline definitions from {Directory}...", + _options.Value.PipelinesDirectory); + + var result = await _registry.ReloadAsync(ct); + + if (!result.Success && _options.Value.StrictPipelineValidation) + { + _logger.LogCritical( + "Pipeline validation failed with {ErrorCount} errors. " + + "Application will stop. Set StrictPipelineValidation=false to allow.", + result.Errors.Count); + + foreach (var error in result.Errors) + { + _logger.LogError( + "Pipeline {Name} ({File}): [{Type}] {Messages}", + error.PipelineName, + error.FileName, + error.ErrorType, + string.Join("; ", error.Messages)); + } + + _lifetime.StopApplication(); + return; + } + + if (result.Errors.Count > 0) + { + _logger.LogWarning( + "Pipeline loading completed with {ErrorCount} warnings", + result.Errors.Count); + + foreach (var error in result.Errors) + { + _logger.LogWarning( + "Pipeline {Name} ({File}): [{Type}] {Messages}", + error.PipelineName, + error.FileName, + error.ErrorType, + string.Join("; ", error.Messages)); + } + } + + _logger.LogInformation( + "Loaded {Count} pipelines ({Enabled} enabled) - version {Version}", + result.PipelinesLoaded, + _registry.GetEnabledPipelines().Count, + result.NewVersion); + } + + /// + public Task StopAsync(CancellationToken ct) => Task.CompletedTask; +} diff --git a/NEW/src/JdeScoping.DataSync/Services/PipelineSnapshot.cs b/NEW/src/JdeScoping.DataSync/Services/PipelineSnapshot.cs new file mode 100644 index 0000000..83f83ad --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Services/PipelineSnapshot.cs @@ -0,0 +1,61 @@ +using JdeScoping.DataSync.Configuration; + +namespace JdeScoping.DataSync.Services; + +/// +/// Immutable snapshot of loaded pipelines. +/// +internal sealed class PipelineSnapshot +{ + /// + /// Empty snapshot with no pipelines. + /// + public static readonly PipelineSnapshot Empty = new([], new PipelineReloadResult { Success = true }); + + private readonly Dictionary _byName; + + /// + /// Gets all loaded pipelines. + /// + public IReadOnlyList AllPipelines { get; } + + /// + /// Gets only enabled pipelines. + /// + public IReadOnlyList EnabledPipelines { get; } + + /// + /// Gets the reload result that created this snapshot. + /// + public PipelineReloadResult Result { get; } + + /// + /// Gets the timestamp when this snapshot was loaded. + /// + public DateTime LoadedAt { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The list of pipelines to include. + /// The reload result. + public PipelineSnapshot(IReadOnlyList pipelines, PipelineReloadResult result) + { + AllPipelines = pipelines; + EnabledPipelines = pipelines.Where(p => p.IsEnabled).ToList(); + Result = result; + LoadedAt = DateTime.UtcNow; + _byName = pipelines.ToDictionary( + p => p.Name, + p => p, + StringComparer.OrdinalIgnoreCase); + } + + /// + /// Gets a pipeline by name (case-insensitive). + /// + /// The pipeline name. + /// The pipeline if found, or null. + public EtlPipelineConfig? GetByName(string name) => + _byName.TryGetValue(name, out var p) ? p : null; +} diff --git a/NEW/src/JdeScoping.DataSync/Services/PipelineValidator.cs b/NEW/src/JdeScoping.DataSync/Services/PipelineValidator.cs new file mode 100644 index 0000000..fea138a --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Services/PipelineValidator.cs @@ -0,0 +1,168 @@ +using JdeScoping.DataSync.Configuration; + +namespace JdeScoping.DataSync.Services; + +/// +/// Validates pipeline definitions according to the schema and business rules. +/// +public class PipelineValidator : IPipelineValidator +{ + private static readonly HashSet ValidConnections = new(StringComparer.OrdinalIgnoreCase) + { + "jde", "cms", "giw", "lotfinder" + }; + + /// + public PipelineValidationResult Validate(EtlPipelineConfig pipeline, string fileName) + { + var result = new PipelineValidationResult { IsValid = true }; + + // Extract expected name from filename (pipeline.{Name}.json) + var expectedName = ExtractPipelineNameFromFileName(fileName); + + // 1. Name must match filename (case-insensitive) + if (!string.Equals(pipeline.Name, expectedName, StringComparison.OrdinalIgnoreCase)) + { + result.Errors.Add($"Pipeline name '{pipeline.Name}' does not match filename '{fileName}' (expected '{expectedName}')"); + } + + // 2. Source is required + if (pipeline.Source == null) + { + result.Errors.Add("Source is required"); + } + else + { + ValidateSource(pipeline.Source, result); + } + + // 3. Destination is required + if (pipeline.Destination == null) + { + result.Errors.Add("Destination is required"); + } + else + { + ValidateDestination(pipeline.Destination, result); + } + + // 4. Interval validation (only for enabled, non-manual-only pipelines) + if (pipeline.IsEnabled && !pipeline.IsManualOnly) + { + ValidateIntervals(pipeline, result); + } + + // 5. Script validation + ValidateScripts(pipeline.PreScripts, "PreScripts", result); + ValidateScripts(pipeline.PostScripts, "PostScripts", result); + + // 6. Warning: MassQuery missing when mass interval set and query has parameters + if (pipeline.MassSyncIntervalMinutes.HasValue && + pipeline.Source?.Parameters?.Count > 0 && + string.IsNullOrEmpty(pipeline.Source.MassQuery)) + { + result.Warnings.Add("MassQuery is not specified but pipeline has parameters and mass sync is enabled. Mass sync will use the incremental query which may not work correctly."); + } + + // 7. Warning: Hourly without daily + if (pipeline.HourlySyncIntervalMinutes.HasValue && !pipeline.DailySyncIntervalMinutes.HasValue) + { + result.Warnings.Add("HourlySyncIntervalMinutes is set without DailySyncIntervalMinutes. Consider adding daily sync."); + } + + result.IsValid = result.Errors.Count == 0; + return result; + } + + private static string ExtractPipelineNameFromFileName(string fileName) + { + // Expected format: pipeline.{Name}.json + if (fileName.StartsWith("pipeline.", StringComparison.OrdinalIgnoreCase) && + fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + return fileName.Substring(9, fileName.Length - 9 - 5); // Remove "pipeline." and ".json" + } + + return fileName; + } + + private static void ValidateSource(SourceElement source, PipelineValidationResult result) + { + // Connection must be valid + if (string.IsNullOrWhiteSpace(source.Connection)) + { + result.Errors.Add("Source.Connection is required"); + } + else if (!ValidConnections.Contains(source.Connection)) + { + result.Errors.Add($"Source.Connection '{source.Connection}' is not valid. Expected one of: {string.Join(", ", ValidConnections)}"); + } + + // Query is required + if (string.IsNullOrWhiteSpace(source.Query)) + { + result.Errors.Add("Source.Query is required"); + } + } + + private static void ValidateDestination(DestinationElement destination, PipelineValidationResult result) + { + // Table is required + if (string.IsNullOrWhiteSpace(destination.Table)) + { + result.Errors.Add("Destination.Table is required"); + } + + // MatchColumns must have at least one entry + if (destination.MatchColumns == null || destination.MatchColumns.Count == 0) + { + result.Errors.Add("Destination.MatchColumns must have at least one column"); + } + } + + private static void ValidateIntervals(EtlPipelineConfig pipeline, PipelineValidationResult result) + { + // At least one interval must be set for enabled, non-manual-only pipelines + var hasAnyInterval = pipeline.MassSyncIntervalMinutes.HasValue || + pipeline.DailySyncIntervalMinutes.HasValue || + pipeline.HourlySyncIntervalMinutes.HasValue; + + if (!hasAnyInterval) + { + result.Errors.Add("At least one sync interval (mass, daily, or hourly) must be set for enabled pipelines. Set IsManualOnly=true for manual-only pipelines."); + } + + // All non-null intervals must be positive + if (pipeline.MassSyncIntervalMinutes.HasValue && pipeline.MassSyncIntervalMinutes.Value <= 0) + { + result.Errors.Add("MassSyncIntervalMinutes must be greater than 0"); + } + + if (pipeline.DailySyncIntervalMinutes.HasValue && pipeline.DailySyncIntervalMinutes.Value <= 0) + { + result.Errors.Add("DailySyncIntervalMinutes must be greater than 0"); + } + + if (pipeline.HourlySyncIntervalMinutes.HasValue && pipeline.HourlySyncIntervalMinutes.Value <= 0) + { + result.Errors.Add("HourlySyncIntervalMinutes must be greater than 0"); + } + } + + private static void ValidateScripts(List? scripts, string scriptType, PipelineValidationResult result) + { + if (scripts == null) + { + return; + } + + for (var i = 0; i < scripts.Count; i++) + { + var script = scripts[i]; + if (string.IsNullOrWhiteSpace(script.Script)) + { + result.Errors.Add($"{scriptType}[{i}].Script is required"); + } + } + } +} diff --git a/NEW/src/JdeScoping.DataSync/Services/ScheduleChecker.cs b/NEW/src/JdeScoping.DataSync/Services/ScheduleChecker.cs index 04d80e5..0296f1d 100644 --- a/NEW/src/JdeScoping.DataSync/Services/ScheduleChecker.cs +++ b/NEW/src/JdeScoping.DataSync/Services/ScheduleChecker.cs @@ -1,6 +1,7 @@ using JdeScoping.Core.Models; using JdeScoping.Core.Models.Enums; using JdeScoping.Core.Models.Infrastructure; +using JdeScoping.DataSync.Configuration; using JdeScoping.DataSync.Options; using JdeScoping.DataSync.Contracts; using JdeScoping.DataSync.Models; @@ -11,10 +12,12 @@ namespace JdeScoping.DataSync.Services; /// /// Checks schedules and determines which sync tasks need to be executed. +/// Uses the pipeline registry for pipeline definitions. /// public class ScheduleChecker : IScheduleChecker { private readonly IDataUpdateRepository _repository; + private readonly IPipelineRegistry _pipelineRegistry; private readonly IOptions _options; private readonly ILogger _logger; @@ -22,14 +25,17 @@ public class ScheduleChecker : IScheduleChecker /// Initializes a new instance of the class. /// /// Repository for data update records. + /// Registry of pipeline definitions. /// Data sync configuration options. /// Logger instance. public ScheduleChecker( IDataUpdateRepository repository, + IPipelineRegistry pipelineRegistry, IOptions options, ILogger logger) { _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + _pipelineRegistry = pipelineRegistry ?? throw new ArgumentNullException(nameof(pipelineRegistry)); _options = options ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -41,9 +47,16 @@ public class ScheduleChecker : IScheduleChecker var tasks = new List(); var now = DateTime.UtcNow; - foreach (var config in _options.Value.DataSources.Where(c => c.IsEnabled)) + // Use the new pipeline registry for scheduling + foreach (var pipeline in _pipelineRegistry.GetEnabledPipelines()) { - var task = CheckConfigSchedule(config, lastUpdates, now); + // Skip manual-only pipelines for scheduled execution + if (pipeline.IsManualOnly) + { + continue; + } + + var task = CheckPipelineSchedule(pipeline, lastUpdates, now); if (task != null) { tasks.Add(task); @@ -66,63 +79,65 @@ public class ScheduleChecker : IScheduleChecker } /// - /// Checks a single data source config and returns a task if sync is needed. + /// Checks a single pipeline and returns a task if sync is needed. /// Priority order: Mass > Daily > Hourly /// - private DataUpdateTask? CheckConfigSchedule( - DataSourceConfig config, + private DataUpdateTask? CheckPipelineSchedule( + EtlPipelineConfig pipeline, Dictionary lastUpdates, DateTime now) { + var tableName = pipeline.Destination.Table; + // Get last updates for each type - var massKey = GetUpdateKey(config.TableName, UpdateTypes.Mass); - var dailyKey = GetUpdateKey(config.TableName, UpdateTypes.Daily); - var hourlyKey = GetUpdateKey(config.TableName, UpdateTypes.Hourly); + var massKey = GetUpdateKey(tableName, UpdateTypes.Mass); + var dailyKey = GetUpdateKey(tableName, UpdateTypes.Daily); + var hourlyKey = GetUpdateKey(tableName, UpdateTypes.Hourly); lastUpdates.TryGetValue(massKey, out var lastMass); lastUpdates.TryGetValue(dailyKey, out var lastDaily); lastUpdates.TryGetValue(hourlyKey, out var lastHourly); // Check Mass first (highest priority) - if (config.MassConfig.Enabled && NeedsMassSync(config, lastMass, now)) + if (pipeline.SupportsMassSync && NeedsMassSync(pipeline, lastMass, now)) { _logger.LogDebug( "Mass sync needed for {Table}: last={LastSync}, interval={Interval}m", - config.TableName, + tableName, lastMass?.EndDt?.ToString("o") ?? "never", - config.MassConfig.IntervalMinutes); + pipeline.MassSyncIntervalMinutes); - return CreateTask(config, UpdateTypes.Mass, null); + return DataUpdateTask.FromPipeline(pipeline, UpdateTypes.Mass); } // Check Daily - if (config.DailyConfig.Enabled && NeedsDailySync(config, lastDaily, lastMass, now)) + if (pipeline.SupportsDailySync && NeedsDailySync(pipeline, lastDaily, lastMass, now)) { - var minimumDt = CalculateMinimumDt(lastDaily, config.DailyConfig.IntervalMinutes); + var minimumDt = CalculateMinimumDt(lastDaily, pipeline.DailySyncIntervalMinutes!.Value); _logger.LogDebug( "Daily sync needed for {Table}: last={LastSync}, interval={Interval}m, minDT={MinDT}", - config.TableName, + tableName, lastDaily?.EndDt?.ToString("o") ?? "never", - config.DailyConfig.IntervalMinutes, + pipeline.DailySyncIntervalMinutes, minimumDt?.ToString("o") ?? "null"); - return CreateTask(config, UpdateTypes.Daily, minimumDt); + return DataUpdateTask.FromPipeline(pipeline, UpdateTypes.Daily, minimumDt); } // Check Hourly - if (config.HourlyConfig.Enabled && NeedsHourlySync(config, lastHourly, lastDaily, lastMass, now)) + if (pipeline.SupportsHourlySync && NeedsHourlySync(pipeline, lastHourly, lastDaily, lastMass, now)) { - var minimumDt = CalculateMinimumDt(lastHourly, config.HourlyConfig.IntervalMinutes); + var minimumDt = CalculateMinimumDt(lastHourly, pipeline.HourlySyncIntervalMinutes!.Value); _logger.LogDebug( "Hourly sync needed for {Table}: last={LastSync}, interval={Interval}m, minDT={MinDT}", - config.TableName, + tableName, lastHourly?.EndDt?.ToString("o") ?? "never", - config.HourlyConfig.IntervalMinutes, + pipeline.HourlySyncIntervalMinutes, minimumDt?.ToString("o") ?? "null"); - return CreateTask(config, UpdateTypes.Hourly, minimumDt); + return DataUpdateTask.FromPipeline(pipeline, UpdateTypes.Hourly, minimumDt); } return null; @@ -131,7 +146,7 @@ public class ScheduleChecker : IScheduleChecker /// /// Determines if a mass sync is needed. /// - private bool NeedsMassSync(DataSourceConfig config, DataUpdate? lastMass, DateTime now) + private static bool NeedsMassSync(EtlPipelineConfig pipeline, DataUpdate? lastMass, DateTime now) { // Never synced before - need mass sync if (lastMass == null) @@ -147,14 +162,18 @@ public class ScheduleChecker : IScheduleChecker } // EndDt is set for successful syncs (GetLastDataUpdatesAsync filters WasSuccessful=1) - var nextSyncDue = lastMass.EndDt!.Value.AddMinutes(config.MassConfig.IntervalMinutes); + var nextSyncDue = lastMass.EndDt!.Value.AddMinutes(pipeline.MassSyncIntervalMinutes!.Value); return now > nextSyncDue; } /// /// Determines if a daily sync is needed. /// - private bool NeedsDailySync(DataSourceConfig config, DataUpdate? lastDaily, DataUpdate? lastMass, DateTime now) + private static bool NeedsDailySync( + EtlPipelineConfig pipeline, + DataUpdate? lastDaily, + DataUpdate? lastMass, + DateTime now) { // If no mass sync ever happened, we need mass first if (lastMass == null) @@ -175,15 +194,15 @@ public class ScheduleChecker : IScheduleChecker } // EndDt is set for successful syncs (GetLastDataUpdatesAsync filters WasSuccessful=1) - var nextSyncDue = lastDaily.EndDt!.Value.AddMinutes(config.DailyConfig.IntervalMinutes); + var nextSyncDue = lastDaily.EndDt!.Value.AddMinutes(pipeline.DailySyncIntervalMinutes!.Value); return now > nextSyncDue; } /// /// Determines if an hourly sync is needed. /// - private bool NeedsHourlySync( - DataSourceConfig config, + private static bool NeedsHourlySync( + EtlPipelineConfig pipeline, DataUpdate? lastHourly, DataUpdate? lastDaily, DataUpdate? lastMass, @@ -208,7 +227,7 @@ public class ScheduleChecker : IScheduleChecker } // EndDt is set for successful syncs (GetLastDataUpdatesAsync filters WasSuccessful=1) - var nextSyncDue = lastHourly.EndDt!.Value.AddMinutes(config.HourlyConfig.IntervalMinutes); + var nextSyncDue = lastHourly.EndDt!.Value.AddMinutes(pipeline.HourlySyncIntervalMinutes!.Value); return now > nextSyncDue; } @@ -227,22 +246,6 @@ public class ScheduleChecker : IScheduleChecker return lastUpdate.EndDt!.Value.AddMinutes(-lookbackMinutes); } - /// - /// Creates a data update task. - /// - private static DataUpdateTask CreateTask(DataSourceConfig config, UpdateTypes updateType, DateTime? minimumDt) - { - return new DataUpdateTask - { - TableName = config.TableName, - SourceSystem = config.SourceSystem, - SourceData = config.SourceData, - UpdateType = updateType, - MinimumDt = minimumDt, - Config = config - }; - } - /// /// Gets the dictionary key for looking up last updates. /// diff --git a/NEW/src/JdeScoping.DataSync/Services/SyncOrchestrator.cs b/NEW/src/JdeScoping.DataSync/Services/SyncOrchestrator.cs index 8793aab..10e7be9 100644 --- a/NEW/src/JdeScoping.DataSync/Services/SyncOrchestrator.cs +++ b/NEW/src/JdeScoping.DataSync/Services/SyncOrchestrator.cs @@ -1,5 +1,6 @@ -using JdeScoping.DataSync.Options; using JdeScoping.DataSync.Contracts; +using JdeScoping.DataSync.Models; +using JdeScoping.DataSync.Options; using JdeScoping.DataSync.Telemetry; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -106,4 +107,53 @@ public class SyncOrchestrator : ISyncOrchestrator _metrics.RecordCycleCompleted(completedCount, failedCount, elapsed.TotalSeconds); } + + /// + public async Task ExecuteSingleSyncAsync(DataUpdateTask task, CancellationToken cancellationToken = default) + { + _logger.LogInformation( + "Executing single sync for {Table} ({Type})", + task.TableName, + task.UpdateType); + + var startTime = DateTime.UtcNow; + + await using var scope = _scopeFactory.CreateAsyncScope(); + + try + { + var operation = scope.ServiceProvider.GetRequiredService(); + await operation.ExecuteAsync(task, cancellationToken); + + var elapsed = DateTime.UtcNow - startTime; + _logger.LogInformation( + "Single sync for {Table} ({Type}) completed in {Elapsed:F1}s", + task.TableName, + task.UpdateType, + elapsed.TotalSeconds); + + _metrics.RecordCycleCompleted(1, 0, elapsed.TotalSeconds); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _logger.LogWarning( + "Single sync for {Table} ({Type}) was cancelled", + task.TableName, + task.UpdateType); + throw; + } + catch (Exception ex) + { + var elapsed = DateTime.UtcNow - startTime; + _logger.LogError( + ex, + "Single sync for {Table} ({Type}) failed after {Elapsed:F1}s", + task.TableName, + task.UpdateType, + elapsed.TotalSeconds); + + _metrics.RecordCycleCompleted(0, 1, elapsed.TotalSeconds); + throw; + } + } } diff --git a/NEW/src/JdeScoping.DataSync/WorkProcessor.cs b/NEW/src/JdeScoping.DataSync/WorkProcessor.cs index fa06e0f..80ffd2b 100644 --- a/NEW/src/JdeScoping.DataSync/WorkProcessor.cs +++ b/NEW/src/JdeScoping.DataSync/WorkProcessor.cs @@ -1,7 +1,12 @@ using JdeScoping.Core.Interfaces; +using JdeScoping.Core.Models.Enums; +using JdeScoping.DataAccess.Services; using JdeScoping.DataSync.Contracts; +using JdeScoping.DataSync.Models; using JdeScoping.DataSync.Options; +using JdeScoping.DataSync.Services; using JdeScoping.DataSync.Telemetry; +using JdeScoping.Domain.Models; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -11,7 +16,7 @@ namespace JdeScoping.DataSync; /// /// Unified background service that coordinates data synchronization and search processing. -/// Data freshness takes priority over search processing. +/// Priority order: Manual sync requests > Scheduled syncs > Search processing. /// public class WorkProcessor : BackgroundService { @@ -52,8 +57,9 @@ public class WorkProcessor : BackgroundService } _logger.LogInformation( - "WorkProcessor starting with WorkInterval={WorkInterval}", - _options.WorkInterval); + "WorkProcessor starting with WorkInterval={WorkInterval}, MaxManualRequestsPerCycle={MaxManual}", + _options.WorkInterval, + _options.MaxManualRequestsPerCycle); // Startup cleanup await StartupCleanupAsync(stoppingToken); @@ -135,13 +141,21 @@ public class WorkProcessor : BackgroundService } /// - /// Performs one work cycle: data syncs have priority, then search processing. + /// Performs one work cycle: manual syncs (priority 1), scheduled syncs (priority 2), search processing (priority 3). /// private async Task DoWorkAsync(CancellationToken ct) { await using var scope = _scopeFactory.CreateAsyncScope(); - // Priority 1: Data syncs + // Priority 1: Manual sync requests (with fairness cap) + var processedManualCount = await ProcessManualSyncRequestsAsync(scope, ct); + if (processedManualCount > 0) + { + // After processing manual requests, still check scheduled syncs + // This implements the fairness policy - manual requests don't completely block scheduled syncs + } + + // Priority 2: Data syncs var scheduleChecker = scope.ServiceProvider.GetRequiredService(); var pendingTasks = await scheduleChecker.GetPendingTasksAsync(ct); @@ -158,7 +172,7 @@ public class WorkProcessor : BackgroundService return "Idle"; } - // Priority 2: Search processing (only when syncs are current) + // Priority 3: Search processing (only when syncs are current) var searchRepository = scope.ServiceProvider.GetRequiredService(); var search = await searchRepository.GetNextQueuedSearchAsync(ct); @@ -176,6 +190,161 @@ public class WorkProcessor : BackgroundService return "Idle"; } + /// + /// Processes pending manual sync requests up to the configured limit. + /// + /// The number of manual requests processed. + private async Task ProcessManualSyncRequestsAsync(AsyncServiceScope scope, CancellationToken ct) + { + var manualSyncService = scope.ServiceProvider.GetService(); + if (manualSyncService == null) + { + // Service not registered - skip manual sync processing + return 0; + } + + var pipelineRegistry = scope.ServiceProvider.GetRequiredService(); + var orchestrator = scope.ServiceProvider.GetRequiredService(); + + var processedCount = 0; + var maxRequests = _options.MaxManualRequestsPerCycle > 0 + ? _options.MaxManualRequestsPerCycle + : int.MaxValue; + + while (processedCount < maxRequests) + { + ct.ThrowIfCancellationRequested(); + + var request = await manualSyncService.GetNextPendingRequestAsync(ct); + if (request == null) + { + break; + } + + await ProcessManualSyncRequestAsync( + request, + pipelineRegistry, + orchestrator, + manualSyncService, + ct); + + processedCount++; + } + + if (processedCount > 0) + { + _logger.LogInformation("Processed {Count} manual sync requests", processedCount); + } + + return processedCount; + } + + /// + /// Processes a single manual sync request. + /// + private async Task ProcessManualSyncRequestAsync( + ManualSyncRequest request, + IPipelineRegistry pipelineRegistry, + ISyncOrchestrator orchestrator, + IManualSyncRequestService manualSyncService, + CancellationToken ct) + { + _logger.LogInformation( + "Processing manual sync request #{Id}: Pipeline={Pipeline}, SyncType={SyncType}", + request.Id, request.PipelineName, request.SyncType); + + await NotifyStatusSafeAsync($"Processing manual sync: {request.PipelineName}", ct); + + try + { + // Get pipeline from registry + var pipeline = pipelineRegistry.GetPipeline(request.PipelineName); + if (pipeline == null) + { + _logger.LogError( + "Manual sync request #{Id} failed: Pipeline '{Name}' not found", + request.Id, request.PipelineName); + // Can't mark as failed without adding a method - just complete it + await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct); + return; + } + + if (!pipeline.IsEnabled) + { + _logger.LogError( + "Manual sync request #{Id} failed: Pipeline '{Name}' is disabled", + request.Id, request.PipelineName); + await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct); + return; + } + + // Parse sync type + if (!TryParseUpdateType(request.SyncType, out var updateType)) + { + _logger.LogError( + "Manual sync request #{Id} failed: Invalid sync type '{Type}'", + request.Id, request.SyncType); + await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct); + return; + } + + // Validate that the pipeline supports this sync type + if (!pipelineRegistry.IsValidPipelineAndSyncType(request.PipelineName, request.SyncType)) + { + _logger.LogError( + "Manual sync request #{Id} failed: Pipeline '{Name}' does not support {Type} sync", + request.Id, request.PipelineName, request.SyncType); + await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct); + return; + } + + // Create and execute the sync task + var task = DataUpdateTask.FromPipeline(pipeline, updateType); + await orchestrator.ExecuteSingleSyncAsync(task, ct); + + // Mark request as completed + await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct); + + _logger.LogInformation( + "Manual sync request #{Id} completed successfully", + request.Id); + } + catch (Exception ex) + { + _logger.LogError(ex, + "Manual sync request #{Id} failed with error", + request.Id); + + // Try to mark as completed (best effort) + try + { + await manualSyncService.CompleteRequestAsync(request.Id, request.RowVersion, ct); + } + catch (Exception completeEx) + { + _logger.LogWarning(completeEx, + "Failed to mark manual sync request #{Id} as completed after error", + request.Id); + } + } + } + + /// + /// Tries to parse a sync type string to an UpdateTypes enum value. + /// + private static bool TryParseUpdateType(string syncType, out UpdateTypes updateType) + { + updateType = syncType.ToLowerInvariant() switch + { + "mass" => UpdateTypes.Mass, + "daily" => UpdateTypes.Daily, + "hourly" => UpdateTypes.Hourly, + _ => (UpdateTypes)(-1) + }; + + return (int)updateType >= 0; + } + /// /// Purges old DataUpdate entries periodically (every 24 hours). /// diff --git a/NEW/src/JdeScoping.Database/Scripts/060_CreateManualSyncRequestTable.sql b/NEW/src/JdeScoping.Database/Scripts/060_CreateManualSyncRequestTable.sql new file mode 100644 index 0000000..19201d0 --- /dev/null +++ b/NEW/src/JdeScoping.Database/Scripts/060_CreateManualSyncRequestTable.sql @@ -0,0 +1,43 @@ +-- Migration: 060_CreateManualSyncRequestTable +-- Purpose: Create ManualSyncRequest table for storing manually requested data syncs + +SET QUOTED_IDENTIFIER ON; +SET ANSI_NULLS ON; +GO + +-- Create ManualSyncRequest table +IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'ManualSyncRequest' AND schema_id = SCHEMA_ID('dbo')) +BEGIN + CREATE TABLE [dbo].[ManualSyncRequest] + ( + [ID] INT IDENTITY(1,1) NOT NULL, + [PipelineName] VARCHAR(128) NOT NULL, + [SyncType] VARCHAR(16) NOT NULL, + [RequestDT] DATETIME2(7) NOT NULL CONSTRAINT [DF_ManualSyncRequest_RequestDT] DEFAULT (SYSUTCDATETIME()), + [RequestedBy] VARCHAR(128) NOT NULL, + [CompletedDT] DATETIME2(7) NULL, + [CancelDT] DATETIME2(7) NULL, + [CancelledBy] VARCHAR(128) NULL, + [RowVersion] ROWVERSION NOT NULL, + CONSTRAINT [PK_ManualSyncRequest] PRIMARY KEY CLUSTERED ([ID]), + CONSTRAINT [CK_ManualSyncRequest_SyncType] CHECK ([SyncType] IN ('mass', 'daily', 'hourly')) + ); +END +GO + +-- Index for pending request queries (FIFO ordering) +IF NOT EXISTS (SELECT * FROM sys.indexes WHERE name = 'IX_ManualSyncRequest_Pending' AND object_id = OBJECT_ID('dbo.ManualSyncRequest')) +BEGIN + CREATE NONCLUSTERED INDEX [IX_ManualSyncRequest_Pending] + ON [dbo].[ManualSyncRequest] ([CompletedDT], [CancelDT], [RequestDT]) + WHERE [CompletedDT] IS NULL AND [CancelDT] IS NULL; +END +GO + +-- Index for user-specific queries +IF NOT EXISTS (SELECT * FROM sys.indexes WHERE name = 'IX_ManualSyncRequest_RequestedBy' AND object_id = OBJECT_ID('dbo.ManualSyncRequest')) +BEGIN + CREATE NONCLUSTERED INDEX [IX_ManualSyncRequest_RequestedBy] + ON [dbo].[ManualSyncRequest] ([RequestedBy], [RequestDT] DESC); +END +GO diff --git a/NEW/src/JdeScoping.Domain/JdeScoping.Domain.csproj b/NEW/src/JdeScoping.Domain/JdeScoping.Domain.csproj new file mode 100644 index 0000000..1b160da --- /dev/null +++ b/NEW/src/JdeScoping.Domain/JdeScoping.Domain.csproj @@ -0,0 +1,9 @@ + + + + net10.0 + enable + enable + + + diff --git a/NEW/src/JdeScoping.Domain/Models/ManualSyncRequest.cs b/NEW/src/JdeScoping.Domain/Models/ManualSyncRequest.cs new file mode 100644 index 0000000..d95a8c3 --- /dev/null +++ b/NEW/src/JdeScoping.Domain/Models/ManualSyncRequest.cs @@ -0,0 +1,59 @@ +namespace JdeScoping.Domain.Models; + +/// +/// Represents a manually requested data sync operation. +/// +public class ManualSyncRequest +{ + /// + /// Gets or sets the unique identifier. + /// + public int Id { get; set; } + + /// + /// Gets or sets the name of the ETL pipeline to sync. + /// + public string PipelineName { get; set; } = string.Empty; + + /// + /// Gets or sets the type of sync (mass, daily, hourly). + /// + public string SyncType { get; set; } = string.Empty; + + /// + /// Gets or sets the UTC timestamp when the request was created. + /// + public DateTime RequestDT { get; set; } + + /// + /// Gets or sets the username of the user who created the request. + /// + public string RequestedBy { get; set; } = string.Empty; + + /// + /// Gets or sets the UTC timestamp when the sync completed, or null if pending. + /// + public DateTime? CompletedDT { get; set; } + + /// + /// Gets or sets the UTC timestamp when the request was cancelled, or null if not cancelled. + /// + public DateTime? CancelDT { get; set; } + + /// + /// Gets or sets the username of the user who cancelled the request, or null. + /// + public string? CancelledBy { get; set; } + + /// + /// Gets or sets the row version for optimistic concurrency. + /// + public byte[] RowVersion { get; set; } = Array.Empty(); + + /// + /// Gets the derived status based on CompletedDT and CancelDT. + /// + public string Status => CancelDT.HasValue ? "Cancelled" + : CompletedDT.HasValue ? "Completed" + : "Pending"; +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.Branch.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.Branch.json new file mode 100644 index 0000000..2aa566a --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.Branch.json @@ -0,0 +1,24 @@ +{ + "name": "Branch", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))", + "massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP'", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "Branch", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.FunctionCode.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.FunctionCode.json new file mode 100644 index 0000000..359d5a6 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.FunctionCode.json @@ -0,0 +1,21 @@ +{ + "name": "FunctionCode", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code", + "massQuery": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code", + "parameters": {} + }, + "destination": { + "table": "FunctionCode", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.Item.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.Item.json new file mode 100644 index 0000000..35fe20e --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.Item.json @@ -0,0 +1,24 @@ +{ + "name": "Item", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL AND (pn.IMUPMJ > :dateUpdated OR (pn.IMUPMJ = :dateUpdated AND pn.IMTDAY >= :timeUpdated))", + "massQuery": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "Item", + "matchColumns": ["ShortItemNumber"], + "excludeFromUpdate": ["ShortItemNumber", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.JdeUser.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.JdeUser.json new file mode 100644 index 0000000..04e0c27 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.JdeUser.json @@ -0,0 +1,21 @@ +{ + "name": "JdeUser", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1", + "massQuery": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1", + "parameters": {} + }, + "destination": { + "table": "JdeUser", + "matchColumns": ["AddressNumber"], + "excludeFromUpdate": ["AddressNumber", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.Lot.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.Lot.json new file mode 100644 index 0000000..9aa4f9d --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.Lot.json @@ -0,0 +1,24 @@ +{ + "name": "Lot", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL AND (lot.IOUPMJ > :dateUpdated OR (lot.IOUPMJ = :dateUpdated AND lot.IOTDAY >= :timeUpdated))", + "massQuery": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "Lot", + "matchColumns": ["LotNumber", "BranchCode"], + "excludeFromUpdate": ["LotNumber", "BranchCode", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.LotUsage_Curr.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.LotUsage_Curr.json new file mode 100644 index 0000000..f1c111a --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.LotUsage_Curr.json @@ -0,0 +1,24 @@ +{ + "name": "LotUsage_Curr", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL AND (lu.ILTRDJ > :dateUpdated OR (lu.ILTRDJ = :dateUpdated AND lu.ILTDAY >= :timeUpdated))", + "massQuery": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "LotUsage_Curr", + "matchColumns": ["UniqueId"], + "excludeFromUpdate": ["UniqueId", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.MisData_Curr.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.MisData_Curr.json new file mode 100644 index 0000000..c99a040 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.MisData_Curr.json @@ -0,0 +1,29 @@ +{ + "name": "MisData_Curr", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 100800, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": null, + "source": { + "connection": "cms", + "query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current' AND Status.PDATE_RELEASED >= :lastUpdateDT", + "massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current'", + "parameters": { + "lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" } + } + }, + "destination": { + "table": "MisData_Curr", + "matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"], + "excludeFromUpdate": [] + }, + "preScripts": [], + "postScripts": [ + { "connection": "lotfinder", "script": "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Curr.MisNumber = bl.MisNumber AND MisData_Curr.RevID = bl.RevID AND MisData_Curr.Status = 'Current' AND bl.Status = 'BackLevel';" }, + { "connection": "lotfinder", "script": "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Curr.MisNumber = nl.MisNumber AND MisData_Curr.RevID < nl.RevID AND MisData_Curr.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;" }, + { "connection": "lotfinder", "script": "MERGE INTO dbo.MisData_Hist AS target USING (SELECT * FROM dbo.MisData_Curr WHERE Status = 'BackLevel') AS source ON target.ItemNumber = source.ItemNumber AND target.BranchCode = source.BranchCode AND target.SequenceNumber = source.SequenceNumber AND target.MisNumber = source.MisNumber AND target.CharNumber = source.CharNumber WHEN MATCHED THEN UPDATE SET target.RevID = source.RevID, target.TestDescription = source.TestDescription, target.SamplingType = source.SamplingType, target.SamplingValue = source.SamplingValue, target.ToolsGauges = source.ToolsGauges, target.WorkInstructions = source.WorkInstructions, target.Status = source.Status, target.ReleaseDate = source.ReleaseDate, target.ObsoleteDate = source.ObsoleteDate WHEN NOT MATCHED THEN INSERT (ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, Status, ReleaseDate, ObsoleteDate) VALUES (source.ItemNumber, source.BranchCode, source.SequenceNumber, source.MisNumber, source.RevID, source.CharNumber, source.TestDescription, source.SamplingType, source.SamplingValue, source.ToolsGauges, source.WorkInstructions, source.Status, source.ReleaseDate, source.ObsoleteDate);" }, + { "connection": "lotfinder", "script": "DELETE FROM dbo.MisData_Curr WHERE Status = 'BackLevel';" }, + { "connection": "lotfinder", "script": "ALTER INDEX [PK_MisData_Curr] ON [dbo].[MisData_Curr] REBUILD;" } + ] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.MisData_Hist.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.MisData_Hist.json new file mode 100644 index 0000000..9bb2bac --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.MisData_Hist.json @@ -0,0 +1,27 @@ +{ + "name": "MisData_Hist", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 100800, + "dailySyncIntervalMinutes": null, + "hourlySyncIntervalMinutes": null, + "source": { + "connection": "cms", + "query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel' AND Status.PDATE_RELEASED >= :lastUpdateDT", + "massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel'", + "parameters": { + "lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" } + } + }, + "destination": { + "table": "MisData_Hist", + "matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"], + "excludeFromUpdate": [] + }, + "preScripts": [], + "postScripts": [ + { "connection": "lotfinder", "script": "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Hist.MisNumber = bl.MisNumber AND MisData_Hist.RevID = bl.RevID AND MisData_Hist.Status = 'Current' AND bl.Status = 'BackLevel';" }, + { "connection": "lotfinder", "script": "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Hist.MisNumber = nl.MisNumber AND MisData_Hist.RevID < nl.RevID AND MisData_Hist.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;" }, + { "connection": "lotfinder", "script": "ALTER INDEX [PK_MisData_Hist] ON [dbo].[MisData_Hist] REBUILD;" } + ] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.OrgHierarchy.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.OrgHierarchy.json new file mode 100644 index 0000000..9347356 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.OrgHierarchy.json @@ -0,0 +1,24 @@ +{ + "name": "OrgHierarchy", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL AND (oh.DATEUPDATED_IWUPMJ > :dateUpdated OR (oh.DATEUPDATED_IWUPMJ = :dateUpdated AND oh.TIMEOFDAY_IWTDAY >= :timeUpdated))", + "massQuery": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "OrgHierarchy", + "matchColumns": ["WorkCenterCode", "BranchCode"], + "excludeFromUpdate": ["WorkCenterCode", "BranchCode", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.ProfitCenter.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.ProfitCenter.json new file mode 100644 index 0000000..503e45a --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.ProfitCenter.json @@ -0,0 +1,24 @@ +{ + "name": "ProfitCenter", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))", + "massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3'", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "ProfitCenter", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.RouteMaster.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.RouteMaster.json new file mode 100644 index 0000000..4f7ab11 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.RouteMaster.json @@ -0,0 +1,24 @@ +{ + "name": "RouteMaster", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL AND (route_master.DATEUPDATED_IRUPMJ > :dateUpdated OR (route_master.DATEUPDATED_IRUPMJ = :dateUpdated AND route_master.TIMEOFDAY_IRTDAY >= :timeUpdated))", + "massQuery": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "RouteMaster", + "matchColumns": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber"], + "excludeFromUpdate": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.StatusCode.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.StatusCode.json new file mode 100644 index 0000000..c3a1c4e --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.StatusCode.json @@ -0,0 +1,24 @@ +{ + "name": "StatusCode", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "giw", + "query": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL AND (sc.DATEUPDATED_DRUPMJ > :dateUpdated OR (sc.DATEUPDATED_DRUPMJ = :dateUpdated AND sc.TIMELASTUPDATED_DRUPMT >= :timeUpdated))", + "massQuery": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "StatusCode", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkCenter.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkCenter.json new file mode 100644 index 0000000..6725883 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkCenter.json @@ -0,0 +1,24 @@ +{ + "name": "WorkCenter", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))", + "massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC'", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "WorkCenter", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderComponent_Curr.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderComponent_Curr.json new file mode 100644 index 0000000..6a81c09 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderComponent_Curr.json @@ -0,0 +1,24 @@ +{ + "name": "WorkOrderComponent_Curr", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL AND (woc.DATEUPDATED_WMUPMJ > :dateUpdated OR (woc.DATEUPDATED_WMUPMJ = :dateUpdated AND woc.TIMEOFDAY_WMTDAY >= :timeUpdated))", + "massQuery": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "WorkOrderComponent_Curr", + "matchColumns": ["UniqueID"], + "excludeFromUpdate": ["UniqueID", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderRouting.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderRouting.json new file mode 100644 index 0000000..340180b --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderRouting.json @@ -0,0 +1,24 @@ +{ + "name": "WorkOrderRouting", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410' AND (woz.DATEUPDATED_SZUPMJ > :dateUpdated OR (woz.DATEUPDATED_SZUPMJ = :dateUpdated AND woz.TIMEOFDAY_SZTDAY >= :timeUpdated))", + "massQuery": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410'", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "WorkOrderRouting", + "matchColumns": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber"], + "excludeFromUpdate": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderStep_Curr.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderStep_Curr.json new file mode 100644 index 0000000..77a3005 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderStep_Curr.json @@ -0,0 +1,24 @@ +{ + "name": "WorkOrderStep_Curr", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY) WHERE (wos.DATEUPDATED_WLUPMJ > :dateUpdated OR (wos.DATEUPDATED_WLUPMJ = :dateUpdated AND wos.TIMEOFDAY_WLTDAY >= :timeUpdated))", + "massQuery": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY)", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "WorkOrderStep_Curr", + "matchColumns": ["WorkOrderNumber", "BranchCode", "StepNumber"], + "excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "StepNumber", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderTime_Curr.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderTime_Curr.json new file mode 100644 index 0000000..dabb569 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrderTime_Curr.json @@ -0,0 +1,24 @@ +{ + "name": "WorkOrderTime_Curr", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot WHERE (wot.DATEUPDATED_WTUPMJ > :dateUpdated OR (wot.DATEUPDATED_WTUPMJ = :dateUpdated AND wot.TIMEOFDAY_WTTDAY >= :timeUpdated))", + "massQuery": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "WorkOrderTime_Curr", + "matchColumns": ["UniqueID"], + "excludeFromUpdate": ["UniqueID", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrder_Curr.json b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrder_Curr.json new file mode 100644 index 0000000..23a4ce3 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipeline.WorkOrder_Curr.json @@ -0,0 +1,24 @@ +{ + "name": "WorkOrder_Curr", + "isEnabled": true, + "isManualOnly": false, + "massSyncIntervalMinutes": 10080, + "dailySyncIntervalMinutes": 1440, + "hourlySyncIntervalMinutes": 60, + "source": { + "connection": "jde", + "query": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo WHERE (wo.WAUPMJ > :dateUpdated OR (wo.WAUPMJ = :dateUpdated AND wo.WATDAY >= :timeUpdated))", + "massQuery": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "destination": { + "table": "WorkOrder_Curr", + "matchColumns": ["WorkOrderNumber", "BranchCode"], + "excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "LastUpdateDt"] + }, + "preScripts": [], + "postScripts": [] +} diff --git a/NEW/src/JdeScoping.Host/appsettings.json b/NEW/src/JdeScoping.Host/appsettings.json index 48be1f6..3d39c51 100644 --- a/NEW/src/JdeScoping.Host/appsettings.json +++ b/NEW/src/JdeScoping.Host/appsettings.json @@ -11,6 +11,8 @@ }, "DataSync": { "Enabled": true, + "PipelinesDirectory": "Pipelines", + "StrictPipelineValidation": true, "CheckInterval": "00:01:00", "MaxDegreeOfParallelism": 8, "BatchSize": 1000000, diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Models/ConnectionStringsSection.cs b/NEW/src/Utils/JdeScoping.ConfigManager/Models/ConnectionStringsSection.cs index d55601b..65a1ea4 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Models/ConnectionStringsSection.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Models/ConnectionStringsSection.cs @@ -1,8 +1,12 @@ +using System.Text.Json.Serialization; + namespace JdeScoping.ConfigManager.Models; /// /// Configuration section for connection strings. +/// Supports both the standard .NET dictionary format and structured entries list. /// +[JsonConverter(typeof(ConnectionStringsSectionConverter))] public class ConnectionStringsSection { public List Entries { get; set; } = new(); diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Models/ConnectionStringsSectionConverter.cs b/NEW/src/Utils/JdeScoping.ConfigManager/Models/ConnectionStringsSectionConverter.cs new file mode 100644 index 0000000..8d3c7f8 --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Models/ConnectionStringsSectionConverter.cs @@ -0,0 +1,293 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace JdeScoping.ConfigManager.Models; + +/// +/// Custom JSON converter that handles the standard .NET ConnectionStrings dictionary format +/// and converts it to a ConnectionStringsSection with an Entries list. +/// +/// +/// Standard appsettings.json uses a dictionary format: +/// +/// "ConnectionStrings": { +/// "LotFinder": "Server=localhost;Database=...;", +/// "JDE": "Data Source=...;" +/// } +/// +/// This converter parses that into a list of ConnectionStringEntry objects. +/// +public class ConnectionStringsSectionConverter : JsonConverter +{ + public override ConnectionStringsSection? Read( + ref Utf8JsonReader reader, + Type typeToConvert, + JsonSerializerOptions options) + { + var section = new ConnectionStringsSection(); + + if (reader.TokenType != JsonTokenType.StartObject) + { + return section; + } + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); + + // Check if this is the "Entries" property (our internal format) + if (string.Equals(propertyName, "entries", StringComparison.OrdinalIgnoreCase)) + { + // Read as array of ConnectionStringEntry + if (reader.TokenType == JsonTokenType.StartArray) + { + section.Entries = JsonSerializer.Deserialize>(ref reader, options) + ?? new List(); + } + } + else if (!string.IsNullOrEmpty(propertyName)) + { + // Standard dictionary format: property name is the connection name, + // value is the connection string + var connectionString = reader.TokenType == JsonTokenType.String + ? reader.GetString() + : null; + + if (!string.IsNullOrEmpty(connectionString)) + { + var entry = ParseConnectionString(propertyName, connectionString); + section.Entries.Add(entry); + } + } + } + } + + return section; + } + + public override void Write( + Utf8JsonWriter writer, + ConnectionStringsSection value, + JsonSerializerOptions options) + { + // Write back in standard dictionary format for compatibility + writer.WriteStartObject(); + + foreach (var entry in value.Entries) + { + if (!string.IsNullOrEmpty(entry.Name)) + { + writer.WriteString(entry.Name, entry.GenerateConnectionString()); + } + } + + writer.WriteEndObject(); + } + + /// + /// Parses a connection string and attempts to detect the provider type. + /// + private static ConnectionStringEntry ParseConnectionString(string name, string connectionString) + { + var entry = new ConnectionStringEntry + { + Name = name, + RawConnectionString = connectionString + }; + + // Try to detect provider and parse structured fields + var parts = ParseConnectionStringParts(connectionString); + + // Detect Oracle first (Data Source with host:port/service pattern) + if (parts.TryGetValue("data source", out var dataSource) && + IsOracleDataSource(dataSource)) + { + entry.Provider = ConnectionProvider.Oracle; + ParseOracleDataSource(entry, dataSource); + + if (parts.TryGetValue("user id", out var oraUserId)) + { + entry.UserId = oraUserId; + } + + if (parts.TryGetValue("password", out var oraPassword)) + { + entry.Password = oraPassword; + } + } + // Detect SQL Server (Server property or Data Source without Oracle pattern) + else if (parts.TryGetValue("server", out var server) || + parts.TryGetValue("data source", out server)) + { + entry.Provider = ConnectionProvider.SqlServer; + entry.Server = server; + + if (parts.TryGetValue("database", out var database) || + parts.TryGetValue("initial catalog", out database)) + { + entry.Database = database; + } + + if (parts.TryGetValue("user id", out var userId) || + parts.TryGetValue("uid", out userId)) + { + entry.UserId = userId; + } + + if (parts.TryGetValue("password", out var password) || + parts.TryGetValue("pwd", out password)) + { + entry.Password = password; + } + + if (parts.TryGetValue("encrypt", out var encrypt)) + { + entry.Encrypt = encrypt; + } + + if (parts.TryGetValue("trustservercertificate", out var trustCert) || + parts.TryGetValue("trust server certificate", out trustCert)) + { + entry.TrustServerCertificate = trustCert.Equals("true", StringComparison.OrdinalIgnoreCase); + } + + if (parts.TryGetValue("connection timeout", out var timeout) || + parts.TryGetValue("connect timeout", out timeout)) + { + if (int.TryParse(timeout, out var timeoutValue)) + { + entry.ConnectionTimeout = timeoutValue; + } + } + + if (parts.TryGetValue("application name", out var appName)) + { + entry.ApplicationName = appName; + } + } + else + { + // Generic/unknown provider - just store raw connection string + entry.Provider = ConnectionProvider.Generic; + } + + return entry; + } + + /// + /// Parses a connection string into key-value pairs. + /// + private static Dictionary ParseConnectionStringParts(string connectionString) + { + var parts = new Dictionary(StringComparer.OrdinalIgnoreCase); + + // Split by semicolon, handling quoted values + var segments = connectionString.Split(';', StringSplitOptions.RemoveEmptyEntries); + + foreach (var segment in segments) + { + var equalsIndex = segment.IndexOf('='); + if (equalsIndex > 0) + { + var key = segment[..equalsIndex].Trim(); + var value = segment[(equalsIndex + 1)..].Trim(); + parts[key] = value; + } + } + + return parts; + } + + /// + /// Determines if a Data Source value looks like an Oracle connection string. + /// Oracle typically uses host:port/service format with numeric port and service name after slash. + /// SQL Server uses server,port or server\instance format. + /// + private static bool IsOracleDataSource(string dataSource) + { + // Check for Oracle patterns: //host:port/service or host:port/service + // SQL Server patterns: server,port or server\instance or just server + + if (string.IsNullOrWhiteSpace(dataSource)) + return false; + + var source = dataSource.TrimStart('/'); + + // Oracle format: has colon followed by port number and then slash + // e.g., "jde-server:1521/JDEPROD" or "//db-host:1523/PRODDB" + var colonIndex = source.IndexOf(':'); + var slashIndex = source.IndexOf('/'); + + if (colonIndex > 0 && slashIndex > colonIndex) + { + // Check if what's between colon and slash is a numeric port + var potentialPort = source[(colonIndex + 1)..slashIndex]; + if (int.TryParse(potentialPort, out _)) + { + return true; + } + } + + // Also check for //host pattern without port + if (dataSource.StartsWith("//")) + { + return true; + } + + return false; + } + + /// + /// Parses Oracle Data Source format (e.g., "//host:port/service" or "host:port/service"). + /// + private static void ParseOracleDataSource(ConnectionStringEntry entry, string dataSource) + { + // Remove leading slashes if present + var source = dataSource.TrimStart('/'); + + // Try to parse host:port/service format + var colonIndex = source.IndexOf(':'); + var slashIndex = source.IndexOf('/'); + + if (colonIndex > 0) + { + entry.Host = source[..colonIndex]; + + if (slashIndex > colonIndex) + { + var portStr = source[(colonIndex + 1)..slashIndex]; + if (int.TryParse(portStr, out var port)) + { + entry.Port = port; + } + + entry.ServiceName = source[(slashIndex + 1)..]; + } + else + { + var portStr = source[(colonIndex + 1)..]; + if (int.TryParse(portStr, out var port)) + { + entry.Port = port; + } + } + } + else if (slashIndex > 0) + { + entry.Host = source[..slashIndex]; + entry.ServiceName = source[(slashIndex + 1)..]; + } + else + { + entry.Host = source; + } + } +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/Forms/PipelineEditorViewModel.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/Forms/PipelineEditorViewModel.cs index eafc021..d79f3e5 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/Forms/PipelineEditorViewModel.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/Forms/PipelineEditorViewModel.cs @@ -1,4 +1,5 @@ using JdeScoping.ConfigManager.Models; +using JdeScoping.ConfigManager.Services; using JdeScoping.ConfigManager.ViewModels.PipelineSteps; using System.Collections.ObjectModel; using System.Windows.Input; @@ -12,14 +13,16 @@ public class PipelineEditorViewModel : ViewModelBase { private readonly PipelineModel _model; private readonly Action _onChanged; + private readonly IDialogService _dialogService; private PipelineStepViewModelBase? _selectedStep; private object? _selectedStepEditor; - public PipelineEditorViewModel(string name, PipelineModel model, IReadOnlyList availableConnections, Action onChanged) + public PipelineEditorViewModel(string name, PipelineModel model, IReadOnlyList availableConnections, IDialogService dialogService, Action onChanged) { Name = name ?? throw new ArgumentNullException(nameof(name)); _model = model ?? throw new ArgumentNullException(nameof(model)); AvailableConnections = availableConnections ?? []; + _dialogService = dialogService ?? throw new ArgumentNullException(nameof(dialogService)); _onChanged = onChanged ?? throw new ArgumentNullException(nameof(onChanged)); // Initialize collections @@ -44,8 +47,11 @@ public class PipelineEditorViewModel : ViewModelBase AddTransformerCommand = new RelayCommand(AddTransformer); AddPostScriptCommand = new RelayCommand(AddPostScript); RemoveStepCommand = new RelayCommand(RemoveStep); + DeleteSelectedStepCommand = new AsyncRelayCommand(DeleteSelectedStepAsync, () => CanDeleteSelectedStep); MoveStepUpCommand = new RelayCommand(MoveStepUp, CanMoveStepUp); MoveStepDownCommand = new RelayCommand(MoveStepDown, CanMoveStepDown); + MoveSelectedStepUpCommand = new RelayCommand(MoveSelectedStepUp, () => CanMoveSelectedStepUp); + MoveSelectedStepDownCommand = new RelayCommand(MoveSelectedStepDown, () => CanMoveSelectedStepDown); } /// @@ -122,11 +128,35 @@ public class PipelineEditorViewModel : ViewModelBase _selectedStep.IsSelected = true; OnPropertyChanged(); + OnPropertyChanged(nameof(CanDeleteSelectedStep)); + OnPropertyChanged(nameof(CanMoveSelectedStepUp)); + OnPropertyChanged(nameof(CanMoveSelectedStepDown)); + (DeleteSelectedStepCommand as AsyncRelayCommand)?.RaiseCanExecuteChanged(); + (MoveSelectedStepUpCommand as RelayCommand)?.RaiseCanExecuteChanged(); + (MoveSelectedStepDownCommand as RelayCommand)?.RaiseCanExecuteChanged(); UpdateSelectedStepEditor(); } } } + /// + /// Gets a value indicating whether the selected step can be deleted. + /// Source and Destination steps cannot be deleted. + /// + public bool CanDeleteSelectedStep => _selectedStep is PreScriptStepViewModel + or TransformerStepViewModelBase + or PostScriptStepViewModel; + + /// + /// Gets a value indicating whether the selected step can be moved up. + /// + public bool CanMoveSelectedStepUp => CanMoveStepUp(_selectedStep); + + /// + /// Gets a value indicating whether the selected step can be moved down. + /// + public bool CanMoveSelectedStepDown => CanMoveStepDown(_selectedStep); + /// /// Gets the editor view model for the currently selected step. /// @@ -156,8 +186,11 @@ public class PipelineEditorViewModel : ViewModelBase public ICommand AddTransformerCommand { get; } public ICommand AddPostScriptCommand { get; } public ICommand RemoveStepCommand { get; } + public ICommand DeleteSelectedStepCommand { get; } public ICommand MoveStepUpCommand { get; } public ICommand MoveStepDownCommand { get; } + public ICommand MoveSelectedStepUpCommand { get; } + public ICommand MoveSelectedStepDownCommand { get; } /// /// Gets the list of available transformer types for the add dialog. @@ -191,7 +224,7 @@ public class PipelineEditorViewModel : ViewModelBase } // Source - Source = new SourceStepViewModel(_model.Source, () => + Source = new SourceStepViewModel(_model.Source, AvailableConnections, () => { _onChanged(); }); @@ -411,6 +444,28 @@ public class PipelineEditorViewModel : ViewModelBase OnPropertyChanged(nameof(AllSteps)); } + private void MoveSelectedStepUp() + { + if (_selectedStep == null) return; + MoveStepUp(_selectedStep); + RaiseMoveCanExecuteChanged(); + } + + private void MoveSelectedStepDown() + { + if (_selectedStep == null) return; + MoveStepDown(_selectedStep); + RaiseMoveCanExecuteChanged(); + } + + private void RaiseMoveCanExecuteChanged() + { + OnPropertyChanged(nameof(CanMoveSelectedStepUp)); + OnPropertyChanged(nameof(CanMoveSelectedStepDown)); + (MoveSelectedStepUpCommand as RelayCommand)?.RaiseCanExecuteChanged(); + (MoveSelectedStepDownCommand as RelayCommand)?.RaiseCanExecuteChanged(); + } + private static void MoveInCollection(ObservableCollection collection, T item, int offset) { var index = collection.IndexOf(item); @@ -440,4 +495,31 @@ public class PipelineEditorViewModel : ViewModelBase ? PostScripts.Select(s => s.Script).ToArray() : null; } + + /// + /// Deletes the currently selected step after user confirmation. + /// Source and Destination steps cannot be deleted. + /// + private async Task DeleteSelectedStepAsync() + { + if (_selectedStep == null || !CanDeleteSelectedStep) + return; + + var stepTypeName = _selectedStep switch + { + PreScriptStepViewModel => "Pre-Script", + TransformerStepViewModelBase t => $"Transformer ({t.TransformerType})", + PostScriptStepViewModel => "Post-Script", + _ => "Step" + }; + + var confirmed = await _dialogService.ShowConfirmationAsync( + "Delete Step", + $"Are you sure you want to delete this {stepTypeName}?\n\nThis action cannot be undone."); + + if (!confirmed) + return; + + RemoveStep(_selectedStep); + } } diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/MainWindowViewModel.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/MainWindowViewModel.cs index 533e0fa..fa4bedd 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/MainWindowViewModel.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/MainWindowViewModel.cs @@ -573,9 +573,9 @@ public class MainWindowViewModel : ViewModelBase MarkAsChanged, _dialogService, _connectionTestService), - _ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null + _ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null && _dialogService != null => _pipelines.Pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline) - ? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), MarkAsChanged) + ? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), _dialogService, MarkAsChanged) : null, _ => null }; diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/SourceStepViewModel.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/SourceStepViewModel.cs index a69aca9..1997fc4 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/SourceStepViewModel.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/SourceStepViewModel.cs @@ -20,9 +20,10 @@ public class SourceStepViewModel : PipelineStepViewModelBase { private readonly PipelineSource _model; - public SourceStepViewModel(PipelineSource model, Action onChanged) : base(onChanged) + public SourceStepViewModel(PipelineSource model, IReadOnlyList availableConnections, Action onChanged) : base(onChanged) { _model = model ?? throw new ArgumentNullException(nameof(model)); + AvailableConnections = availableConnections ?? []; // Initialize parameters collection Parameters = new ObservableCollection( @@ -36,6 +37,11 @@ public class SourceStepViewModel : PipelineStepViewModelBase AddParameterCommand = new RelayCommand(AddParameter); } + /// + /// Gets the list of available connection string names from configuration. + /// + public IReadOnlyList AvailableConnections { get; } + public override PipelineStepType StepType => PipelineStepType.Source; public override string DisplayName => "Source"; public override string Icon => IsFileSource ? "󰈔" : "󰆼"; // mdi-file vs mdi-database diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Editors/SourceEditorView.axaml b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Editors/SourceEditorView.axaml index 457d57f..112e88f 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Editors/SourceEditorView.axaml +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Editors/SourceEditorView.axaml @@ -39,12 +39,13 @@ Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/> - - + diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/ConnectionStringsFormView.axaml b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/ConnectionStringsFormView.axaml index 08bbbd1..c1d50fe 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/ConnectionStringsFormView.axaml +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/ConnectionStringsFormView.axaml @@ -279,40 +279,71 @@ - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/PipelineEditorView.axaml b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/PipelineEditorView.axaml index 6da3bb8..1e28948 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/PipelineEditorView.axaml +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/PipelineEditorView.axaml @@ -188,12 +188,49 @@ - - - - - + + + + + + + + + + + + + + + + + diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/MainWindow.axaml b/NEW/src/Utils/JdeScoping.ConfigManager/Views/MainWindow.axaml index 31bb227..f887d03 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Views/MainWindow.axaml +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/MainWindow.axaml @@ -36,6 +36,9 @@ + + + diff --git a/NEW/tests/JdeScoping.Api.Tests/Controllers/ManualSyncControllerTests.cs b/NEW/tests/JdeScoping.Api.Tests/Controllers/ManualSyncControllerTests.cs new file mode 100644 index 0000000..93e3080 --- /dev/null +++ b/NEW/tests/JdeScoping.Api.Tests/Controllers/ManualSyncControllerTests.cs @@ -0,0 +1,402 @@ +using System.Security.Claims; +using JdeScoping.Api.Contracts.ManualSync; +using JdeScoping.Api.Controllers; +using JdeScoping.DataAccess.Services; +using JdeScoping.DataSync.Configuration; +using JdeScoping.DataSync.Services; +using JdeScoping.Domain.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using NSubstitute; +using Shouldly; +using Xunit; + +namespace JdeScoping.Api.Tests.Controllers; + +public class ManualSyncControllerTests +{ + private readonly IManualSyncRequestService _manualSyncRequestService; + private readonly IPipelineRegistry _pipelineRegistry; + private readonly ManualSyncController _controller; + + public ManualSyncControllerTests() + { + _manualSyncRequestService = Substitute.For(); + _pipelineRegistry = Substitute.For(); + _controller = new ManualSyncController(_manualSyncRequestService, _pipelineRegistry); + SetupAuthenticatedUser("testuser"); + } + + #region GetRequests Tests + + [Fact] + public async Task GetRequests_ReturnsOkWithRequests() + { + // Arrange + var requests = new List + { + CreateRequest(1, "Pipeline1", "mass", "user1", DateTime.UtcNow.AddHours(-2)), + CreateRequest(2, "Pipeline2", "daily", "user2", DateTime.UtcNow.AddHours(-1)) + }; + _manualSyncRequestService.GetRequestsAsync(false, Arg.Any()) + .Returns(requests); + + // Act + var result = await _controller.GetRequests(false, CancellationToken.None); + + // Assert + result.Result.ShouldBeOfType(); + var okResult = (OkObjectResult)result.Result!; + var viewModels = okResult.Value.ShouldBeAssignableTo>()!; + viewModels.Count.ShouldBe(2); + viewModels[0].Id.ShouldBe(1); + viewModels[0].PipelineName.ShouldBe("Pipeline1"); + viewModels[1].Id.ShouldBe(2); + viewModels[1].PipelineName.ShouldBe("Pipeline2"); + } + + [Fact] + public async Task GetRequests_WithPendingOnlyTrue_PassesPendingOnlyToService() + { + // Arrange + _manualSyncRequestService.GetRequestsAsync(true, Arg.Any()) + .Returns(new List()); + + // Act + await _controller.GetRequests(true, CancellationToken.None); + + // Assert + await _manualSyncRequestService.Received(1) + .GetRequestsAsync(true, Arg.Any()); + } + + [Fact] + public async Task GetRequests_MapsRowVersionToBase64() + { + // Arrange + var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; + var request = CreateRequest(1, "Pipeline1", "mass", "user1", DateTime.UtcNow, rowVersion: rowVersion); + _manualSyncRequestService.GetRequestsAsync(false, Arg.Any()) + .Returns(new List { request }); + + // Act + var result = await _controller.GetRequests(false, CancellationToken.None); + + // Assert + var okResult = (OkObjectResult)result.Result!; + var viewModels = okResult.Value.ShouldBeAssignableTo>()!; + viewModels[0].RowVersionBase64.ShouldBe(Convert.ToBase64String(rowVersion)); + } + + #endregion + + #region GetPipelines Tests + + [Fact] + public void GetPipelines_ReturnsOkWithPipelines() + { + // Arrange + var pipelines = new List + { + CreatePipeline("WorkOrders", massSyncInterval: 1440, dailySyncInterval: 60, hourlySyncInterval: 15), + CreatePipeline("Items", massSyncInterval: 1440, dailySyncInterval: 60) + }; + _pipelineRegistry.GetEnabledPipelines().Returns(pipelines); + + // Act + var result = _controller.GetPipelines(); + + // Assert + result.Result.ShouldBeOfType(); + var okResult = (OkObjectResult)result.Result!; + var viewModels = okResult.Value.ShouldBeAssignableTo>()!; + viewModels.Count.ShouldBe(2); + viewModels[0].Name.ShouldBe("WorkOrders"); + viewModels[0].SupportedSyncTypes.ShouldContain("mass"); + viewModels[0].SupportedSyncTypes.ShouldContain("daily"); + viewModels[0].SupportedSyncTypes.ShouldContain("hourly"); + viewModels[1].Name.ShouldBe("Items"); + } + + [Fact] + public void GetPipelines_WhenEmpty_ReturnsEmptyList() + { + // Arrange + _pipelineRegistry.GetEnabledPipelines().Returns(new List()); + + // Act + var result = _controller.GetPipelines(); + + // Assert + var okResult = (OkObjectResult)result.Result!; + var viewModels = okResult.Value.ShouldBeAssignableTo>()!; + viewModels.ShouldBeEmpty(); + } + + #endregion + + #region CreateRequest Tests + + [Fact] + public async Task CreateRequest_WithValidInput_ReturnsCreated() + { + // Arrange + var dto = new CreateManualSyncRequestDto + { + PipelineName = "WorkOrders", + SyncType = "mass" + }; + var createdRequest = CreateRequest(42, "WorkOrders", "mass", "testuser", DateTime.UtcNow); + + _pipelineRegistry.IsValidPipelineAndSyncType("WorkOrders", "mass").Returns(true); + _manualSyncRequestService.CreateRequestAsync("WorkOrders", "mass", "testuser", Arg.Any()) + .Returns(createdRequest); + + // Act + var result = await _controller.CreateRequest(dto, CancellationToken.None); + + // Assert + result.Result.ShouldBeOfType(); + var createdResult = (CreatedAtActionResult)result.Result!; + var viewModel = createdResult.Value.ShouldBeOfType(); + viewModel.Id.ShouldBe(42); + viewModel.PipelineName.ShouldBe("WorkOrders"); + viewModel.SyncType.ShouldBe("mass"); + viewModel.RequestedBy.ShouldBe("testuser"); + } + + [Fact] + public async Task CreateRequest_WithInvalidPipelineOrSyncType_ReturnsBadRequest() + { + // Arrange + var dto = new CreateManualSyncRequestDto + { + PipelineName = "InvalidPipeline", + SyncType = "invalid" + }; + _pipelineRegistry.IsValidPipelineAndSyncType("InvalidPipeline", "invalid").Returns(false); + + // Act + var result = await _controller.CreateRequest(dto, CancellationToken.None); + + // Assert + result.Result.ShouldBeOfType(); + var badRequestResult = (BadRequestObjectResult)result.Result!; + badRequestResult.Value.ShouldBeOfType(); + ((string)badRequestResult.Value!).ShouldContain("Invalid pipeline/sync type combination"); + } + + [Fact] + public async Task CreateRequest_WhenUnauthenticated_ReturnsUnauthorized() + { + // Arrange + SetupUnauthenticatedUser(); + var dto = new CreateManualSyncRequestDto + { + PipelineName = "WorkOrders", + SyncType = "mass" + }; + + // Act + var result = await _controller.CreateRequest(dto, CancellationToken.None); + + // Assert + result.Result.ShouldBeOfType(); + } + + [Fact] + public async Task CreateRequest_PassesCorrectUsernameToService() + { + // Arrange + var dto = new CreateManualSyncRequestDto + { + PipelineName = "WorkOrders", + SyncType = "daily" + }; + var createdRequest = CreateRequest(1, "WorkOrders", "daily", "testuser", DateTime.UtcNow); + + _pipelineRegistry.IsValidPipelineAndSyncType("WorkOrders", "daily").Returns(true); + _manualSyncRequestService.CreateRequestAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(createdRequest); + + // Act + await _controller.CreateRequest(dto, CancellationToken.None); + + // Assert + await _manualSyncRequestService.Received(1) + .CreateRequestAsync("WorkOrders", "daily", "testuser", Arg.Any()); + } + + #endregion + + #region CancelRequest Tests + + [Fact] + public async Task CancelRequest_WhenSuccessful_ReturnsOk() + { + // Arrange + var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; + var dto = new CancelManualSyncRequestDto + { + RowVersionBase64 = Convert.ToBase64String(rowVersion) + }; + _manualSyncRequestService.CancelRequestAsync(1, "testuser", Arg.Any(), Arg.Any()) + .Returns(true); + + // Act + var result = await _controller.CancelRequest(1, dto, CancellationToken.None); + + // Assert + result.ShouldBeOfType(); + } + + [Fact] + public async Task CancelRequest_WhenConcurrencyFails_ReturnsConflict() + { + // Arrange + var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; + var dto = new CancelManualSyncRequestDto + { + RowVersionBase64 = Convert.ToBase64String(rowVersion) + }; + _manualSyncRequestService.CancelRequestAsync(1, "testuser", Arg.Any(), Arg.Any()) + .Returns(false); + + // Act + var result = await _controller.CancelRequest(1, dto, CancellationToken.None); + + // Assert + result.ShouldBeOfType(); + } + + [Fact] + public async Task CancelRequest_WithInvalidBase64_ReturnsBadRequest() + { + // Arrange + var dto = new CancelManualSyncRequestDto + { + RowVersionBase64 = "not-valid-base64!!!" + }; + + // Act + var result = await _controller.CancelRequest(1, dto, CancellationToken.None); + + // Assert + result.ShouldBeOfType(); + var badRequest = (BadRequestObjectResult)result; + badRequest.Value.ShouldBeOfType(); + ((string)badRequest.Value!).ShouldContain("Invalid RowVersionBase64 format"); + } + + [Fact] + public async Task CancelRequest_WhenUnauthenticated_ReturnsUnauthorized() + { + // Arrange + SetupUnauthenticatedUser(); + var dto = new CancelManualSyncRequestDto + { + RowVersionBase64 = Convert.ToBase64String(new byte[] { 1, 2, 3 }) + }; + + // Act + var result = await _controller.CancelRequest(1, dto, CancellationToken.None); + + // Assert + result.ShouldBeOfType(); + } + + [Fact] + public async Task CancelRequest_PassesCorrectParametersToService() + { + // Arrange + var rowVersion = new byte[] { 10, 20, 30, 40 }; + var dto = new CancelManualSyncRequestDto + { + RowVersionBase64 = Convert.ToBase64String(rowVersion) + }; + _manualSyncRequestService.CancelRequestAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(true); + + // Act + await _controller.CancelRequest(99, dto, CancellationToken.None); + + // Assert + await _manualSyncRequestService.Received(1) + .CancelRequestAsync( + 99, + "testuser", + Arg.Is(b => b.SequenceEqual(rowVersion)), + Arg.Any()); + } + + #endregion + + #region Helper Methods + + private void SetupAuthenticatedUser(string username) + { + var claims = new List + { + new(ClaimTypes.Name, username), + new("dn", $"CN={username},DC=example,DC=com") + }; + var identity = new ClaimsIdentity(claims, "Test"); + var principal = new ClaimsPrincipal(identity); + + var httpContext = new DefaultHttpContext { User = principal }; + _controller.ControllerContext = new ControllerContext { HttpContext = httpContext }; + } + + private void SetupUnauthenticatedUser() + { + var identity = new ClaimsIdentity(); // No claims, not authenticated + var principal = new ClaimsPrincipal(identity); + + var httpContext = new DefaultHttpContext { User = principal }; + _controller.ControllerContext = new ControllerContext { HttpContext = httpContext }; + } + + private static ManualSyncRequest CreateRequest( + int id, + string pipelineName, + string syncType, + string requestedBy, + DateTime requestDT, + DateTime? completedDT = null, + DateTime? cancelDT = null, + string? cancelledBy = null, + byte[]? rowVersion = null) + { + return new ManualSyncRequest + { + Id = id, + PipelineName = pipelineName, + SyncType = syncType, + RequestedBy = requestedBy, + RequestDT = requestDT, + CompletedDT = completedDT, + CancelDT = cancelDT, + CancelledBy = cancelledBy, + RowVersion = rowVersion ?? new byte[] { 0, 0, 0, 0, 0, 0, 0, 1 } + }; + } + + private static EtlPipelineConfig CreatePipeline( + string name, + int? massSyncInterval = null, + int? dailySyncInterval = null, + int? hourlySyncInterval = null, + bool isEnabled = true) + { + return new EtlPipelineConfig + { + Name = name, + IsEnabled = isEnabled, + MassSyncIntervalMinutes = massSyncInterval, + DailySyncIntervalMinutes = dailySyncInterval, + HourlySyncIntervalMinutes = hourlySyncInterval + }; + } + + #endregion +} diff --git a/NEW/tests/JdeScoping.Api.Tests/Controllers/PipelineControllerTests.cs b/NEW/tests/JdeScoping.Api.Tests/Controllers/PipelineControllerTests.cs new file mode 100644 index 0000000..77f7bd9 --- /dev/null +++ b/NEW/tests/JdeScoping.Api.Tests/Controllers/PipelineControllerTests.cs @@ -0,0 +1,289 @@ +using System.Security.Claims; +using JdeScoping.Api.Controllers; +using JdeScoping.DataSync.Configuration; +using JdeScoping.DataSync.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using NSubstitute; +using NSubstitute.ExceptionExtensions; +using Shouldly; +using Xunit; + +namespace JdeScoping.Api.Tests.Controllers; + +public class PipelineControllerTests +{ + private readonly IPipelineRegistry _pipelineRegistry; + private readonly ILogger _logger; + private readonly PipelineController _controller; + + public PipelineControllerTests() + { + _pipelineRegistry = Substitute.For(); + _logger = Substitute.For>(); + _controller = new PipelineController(_pipelineRegistry, _logger); + SetupAuthenticatedUser("testuser", isAdmin: false); + } + + #region GetPipelines Tests + + [Fact] + public void GetPipelines_ReturnsEnabledPipelines() + { + // Arrange + var pipelines = new List + { + CreatePipeline("WorkOrders", massSyncInterval: 1440, dailySyncInterval: 60, hourlySyncInterval: 15), + CreatePipeline("Items", massSyncInterval: 1440, dailySyncInterval: 60) + }; + _pipelineRegistry.GetEnabledPipelines().Returns(pipelines); + + // Act + var result = _controller.GetPipelines(); + + // Assert + result.Result.ShouldBeOfType(); + var okResult = (OkObjectResult)result.Result!; + var viewModels = okResult.Value.ShouldBeAssignableTo>()!; + + viewModels.Count.ShouldBe(2); + viewModels[0].Name.ShouldBe("WorkOrders"); + viewModels[0].SupportedSyncTypes.ShouldContain("mass"); + viewModels[0].SupportedSyncTypes.ShouldContain("daily"); + viewModels[0].SupportedSyncTypes.ShouldContain("hourly"); + viewModels[1].Name.ShouldBe("Items"); + viewModels[1].SupportedSyncTypes.ShouldNotContain("hourly"); + } + + [Fact] + public void GetPipelines_WhenEmpty_ReturnsEmptyList() + { + // Arrange + _pipelineRegistry.GetEnabledPipelines().Returns(new List()); + + // Act + var result = _controller.GetPipelines(); + + // Assert + var okResult = (OkObjectResult)result.Result!; + var viewModels = okResult.Value.ShouldBeAssignableTo>()!; + viewModels.ShouldBeEmpty(); + } + + [Fact] + public void GetPipelines_MapsOnlySupportedSyncTypes() + { + // Arrange - pipeline with only mass sync + var pipelines = new List + { + CreatePipeline("MassOnly", massSyncInterval: 1440) + }; + _pipelineRegistry.GetEnabledPipelines().Returns(pipelines); + + // Act + var result = _controller.GetPipelines(); + + // Assert + var okResult = (OkObjectResult)result.Result!; + var viewModels = okResult.Value.ShouldBeAssignableTo>()!; + + viewModels[0].SupportedSyncTypes.Count.ShouldBe(1); + viewModels[0].SupportedSyncTypes.ShouldContain("mass"); + } + + #endregion + + #region GetStatus Tests + + [Fact] + public void GetStatus_ReturnsRegistryMetadata() + { + // Arrange + var allPipelines = new List + { + CreatePipeline("Pipeline1", isEnabled: true), + CreatePipeline("Pipeline2", isEnabled: true), + CreatePipeline("Pipeline3", isEnabled: false) + }; + var enabledPipelines = allPipelines.Where(p => p.IsEnabled).ToList(); + + _pipelineRegistry.GetAllPipelines().Returns(allPipelines); + _pipelineRegistry.GetEnabledPipelines().Returns(enabledPipelines); + _pipelineRegistry.Version.Returns(5); + _pipelineRegistry.LastLoadedAt.Returns(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc)); + + // Act + var result = _controller.GetStatus(); + + // Assert + result.Result.ShouldBeOfType(); + var okResult = (OkObjectResult)result.Result!; + var status = okResult.Value.ShouldBeOfType(); + + status.Version.ShouldBe(5); + status.LastLoadedAt.ShouldBe(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc)); + status.TotalPipelines.ShouldBe(3); + status.EnabledPipelines.ShouldBe(2); + } + + #endregion + + #region ReloadPipelines Tests + + [Fact] + public async Task ReloadPipelines_CallsRegistry() + { + // Arrange + SetupAuthenticatedUser("admin", isAdmin: true); + _pipelineRegistry.ReloadAsync(Arg.Any()) + .Returns(CreateSuccessfulReloadResult(5, 0)); + + // Act + await _controller.ReloadPipelines(CancellationToken.None); + + // Assert + await _pipelineRegistry.Received(1).ReloadAsync(Arg.Any()); + } + + [Fact] + public async Task ReloadPipelines_ReturnsResult() + { + // Arrange + SetupAuthenticatedUser("admin", isAdmin: true); + _pipelineRegistry.ReloadAsync(Arg.Any()) + .Returns(CreateSuccessfulReloadResult(pipelinesLoaded: 10, pipelinesSkipped: 2, previousVersion: 3, newVersion: 4)); + + // Act + var result = await _controller.ReloadPipelines(CancellationToken.None); + + // Assert + result.Result.ShouldBeOfType(); + var okResult = (OkObjectResult)result.Result!; + var reloadResult = okResult.Value.ShouldBeOfType(); + + reloadResult.Success.ShouldBeTrue(); + reloadResult.PipelinesLoaded.ShouldBe(10); + reloadResult.PipelinesSkipped.ShouldBe(2); + reloadResult.PreviousVersion.ShouldBe(3); + reloadResult.NewVersion.ShouldBe(4); + } + + [Fact] + public async Task ReloadPipelines_WithErrors_ReturnsErrorDetails() + { + // Arrange + SetupAuthenticatedUser("admin", isAdmin: true); + var errors = new List + { + new PipelineLoadError + { + FileName = "pipeline.Bad.json", + PipelineName = "Bad", + ErrorType = "Validation", + Messages = new List { "Missing source", "Missing destination" } + } + }; + + _pipelineRegistry.ReloadAsync(Arg.Any()) + .Returns(new PipelineReloadResult + { + Success = false, + PipelinesLoaded = 5, + PipelinesSkipped = 1, + PreviousVersion = 1, + NewVersion = 1, + Errors = errors + }); + + // Act + var result = await _controller.ReloadPipelines(CancellationToken.None); + + // Assert + result.Result.ShouldBeOfType(); + var okResult = (OkObjectResult)result.Result!; + var reloadResult = okResult.Value.ShouldBeOfType(); + + reloadResult.Success.ShouldBeFalse(); + reloadResult.Errors.Count.ShouldBe(1); + reloadResult.Errors[0].FileName.ShouldBe("pipeline.Bad.json"); + reloadResult.Errors[0].ErrorType.ShouldBe("Validation"); + reloadResult.Errors[0].Messages.ShouldContain("Missing source"); + } + + [Fact] + public async Task ReloadPipelines_WhenExceptionThrown_Returns500() + { + // Arrange + SetupAuthenticatedUser("admin", isAdmin: true); + _pipelineRegistry.ReloadAsync(Arg.Any()) + .Throws(new Exception("Unexpected error")); + + // Act + var result = await _controller.ReloadPipelines(CancellationToken.None); + + // Assert + result.Result.ShouldBeOfType(); + var objectResult = (ObjectResult)result.Result!; + objectResult.StatusCode.ShouldBe(StatusCodes.Status500InternalServerError); + } + + #endregion + + #region Helper Methods + + private void SetupAuthenticatedUser(string username, bool isAdmin = false) + { + var claims = new List + { + new(ClaimTypes.Name, username) + }; + + if (isAdmin) + { + claims.Add(new Claim(ClaimTypes.Role, "Admin")); + } + + var identity = new ClaimsIdentity(claims, "Test"); + var principal = new ClaimsPrincipal(identity); + + var httpContext = new DefaultHttpContext { User = principal }; + _controller.ControllerContext = new ControllerContext { HttpContext = httpContext }; + } + + private static EtlPipelineConfig CreatePipeline( + string name, + bool isEnabled = true, + int? massSyncInterval = null, + int? dailySyncInterval = null, + int? hourlySyncInterval = null) + { + return new EtlPipelineConfig + { + Name = name, + IsEnabled = isEnabled, + MassSyncIntervalMinutes = massSyncInterval, + DailySyncIntervalMinutes = dailySyncInterval, + HourlySyncIntervalMinutes = hourlySyncInterval + }; + } + + private static PipelineReloadResult CreateSuccessfulReloadResult( + int pipelinesLoaded, + int pipelinesSkipped, + int previousVersion = 1, + int newVersion = 2) + { + return new PipelineReloadResult + { + Success = true, + PipelinesLoaded = pipelinesLoaded, + PipelinesSkipped = pipelinesSkipped, + PreviousVersion = previousVersion, + NewVersion = newVersion, + Errors = new List() + }; + } + + #endregion +} diff --git a/NEW/tests/JdeScoping.ConfigManager.Tests/Models/ConnectionStringsSectionConverterTests.cs b/NEW/tests/JdeScoping.ConfigManager.Tests/Models/ConnectionStringsSectionConverterTests.cs new file mode 100644 index 0000000..5265a09 --- /dev/null +++ b/NEW/tests/JdeScoping.ConfigManager.Tests/Models/ConnectionStringsSectionConverterTests.cs @@ -0,0 +1,198 @@ +using System.Text.Json; +using JdeScoping.ConfigManager.Models; +using Shouldly; +using Xunit; + +namespace JdeScoping.ConfigManager.Tests.Models; + +public class ConnectionStringsSectionConverterTests +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true + }; + + [Fact] + public void Deserialize_StandardDictionaryFormat_ParsesAllConnections() + { + // Arrange + var json = """ + { + "LotFinder": "Server=localhost,1434;Database=ScopingTool;User Id=sa;Password=test", + "JDE": "Data Source=jde-server:1521/JDEPROD;User Id=jdeuser;Password=jdepass", + "CMS": "Data Source=cms-server:1521/CMSPROD;User Id=cmsuser;Password=cmspass" + } + """; + + // Act + var section = JsonSerializer.Deserialize(json, JsonOptions); + + // Assert + section.ShouldNotBeNull(); + section.Entries.Count.ShouldBe(3); + + var lotFinder = section.Entries.First(e => e.Name == "LotFinder"); + lotFinder.Provider.ShouldBe(ConnectionProvider.SqlServer); + lotFinder.Server.ShouldBe("localhost,1434"); + lotFinder.Database.ShouldBe("ScopingTool"); + lotFinder.UserId.ShouldBe("sa"); + + var jde = section.Entries.First(e => e.Name == "JDE"); + jde.Provider.ShouldBe(ConnectionProvider.Oracle); + jde.Host.ShouldBe("jde-server"); + jde.Port.ShouldBe(1521); + jde.ServiceName.ShouldBe("JDEPROD"); + } + + [Fact] + public void Deserialize_SqlServerConnection_ParsesAllFields() + { + // Arrange + var json = """ + { + "TestDb": "Server=myserver;Database=TestDB;User Id=testuser;Password=testpass;Encrypt=True;TrustServerCertificate=True;Connection Timeout=60;Application Name=TestApp" + } + """; + + // Act + var section = JsonSerializer.Deserialize(json, JsonOptions); + + // Assert + section.ShouldNotBeNull(); + section.Entries.Count.ShouldBe(1); + + var entry = section.Entries[0]; + entry.Name.ShouldBe("TestDb"); + entry.Provider.ShouldBe(ConnectionProvider.SqlServer); + entry.Server.ShouldBe("myserver"); + entry.Database.ShouldBe("TestDB"); + entry.UserId.ShouldBe("testuser"); + entry.Password.ShouldBe("testpass"); + entry.Encrypt.ShouldBe("True"); + entry.TrustServerCertificate.ShouldBeTrue(); + entry.ConnectionTimeout.ShouldBe(60); + entry.ApplicationName.ShouldBe("TestApp"); + } + + [Fact] + public void Deserialize_OracleConnection_ParsesHostPortService() + { + // Arrange + var json = """ + { + "Oracle": "Data Source=//db-host:1523/PRODDB;User Id=orauser;Password=orapass" + } + """; + + // Act + var section = JsonSerializer.Deserialize(json, JsonOptions); + + // Assert + section.ShouldNotBeNull(); + section.Entries.Count.ShouldBe(1); + + var entry = section.Entries[0]; + entry.Name.ShouldBe("Oracle"); + entry.Provider.ShouldBe(ConnectionProvider.Oracle); + entry.Host.ShouldBe("db-host"); + entry.Port.ShouldBe(1523); + entry.ServiceName.ShouldBe("PRODDB"); + entry.UserId.ShouldBe("orauser"); + entry.Password.ShouldBe("orapass"); + } + + [Fact] + public void Deserialize_EmptyObject_ReturnsEmptyEntries() + { + // Arrange + var json = "{}"; + + // Act + var section = JsonSerializer.Deserialize(json, JsonOptions); + + // Assert + section.ShouldNotBeNull(); + section.Entries.ShouldBeEmpty(); + } + + [Fact] + public void Serialize_ToStandardDictionaryFormat() + { + // Arrange + var section = new ConnectionStringsSection + { + Entries = new List + { + new() + { + Name = "TestDb", + Provider = ConnectionProvider.SqlServer, + Server = "myserver", + Database = "TestDB", + UserId = "testuser", + Password = "testpass" + } + } + }; + + // Act + var json = JsonSerializer.Serialize(section, JsonOptions); + + // Assert + json.ShouldNotBeNull(); + json.ShouldContain("\"TestDb\""); + json.ShouldContain("Server=myserver"); + json.ShouldContain("Database=TestDB"); + } + + [Fact] + public void RoundTrip_PreservesConnections() + { + // Arrange + var original = new ConnectionStringsSection + { + Entries = new List + { + new() + { + Name = "Primary", + Provider = ConnectionProvider.SqlServer, + Server = "server1", + Database = "DB1", + UserId = "user1", + Password = "pass1" + }, + new() + { + Name = "Secondary", + Provider = ConnectionProvider.Oracle, + Host = "oracle-host", + Port = 1521, + ServiceName = "ORCL", + UserId = "user2", + Password = "pass2" + } + } + }; + + // Act + var json = JsonSerializer.Serialize(original, JsonOptions); + var deserialized = JsonSerializer.Deserialize(json, JsonOptions); + + // Assert + deserialized.ShouldNotBeNull(); + deserialized.Entries.Count.ShouldBe(2); + + var primary = deserialized.Entries.First(e => e.Name == "Primary"); + primary.Provider.ShouldBe(ConnectionProvider.SqlServer); + primary.Server.ShouldBe("server1"); + primary.Database.ShouldBe("DB1"); + + var secondary = deserialized.Entries.First(e => e.Name == "Secondary"); + secondary.Provider.ShouldBe(ConnectionProvider.Oracle); + secondary.Host.ShouldBe("oracle-host"); + secondary.Port.ShouldBe(1521); + secondary.ServiceName.ShouldBe("ORCL"); + } +} diff --git a/NEW/tests/JdeScoping.DataAccess.Tests/Services/ManualSyncRequestServiceTests.cs b/NEW/tests/JdeScoping.DataAccess.Tests/Services/ManualSyncRequestServiceTests.cs new file mode 100644 index 0000000..bba802b --- /dev/null +++ b/NEW/tests/JdeScoping.DataAccess.Tests/Services/ManualSyncRequestServiceTests.cs @@ -0,0 +1,382 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataAccess.Services; +using JdeScoping.Domain.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; +using Xunit; + +namespace JdeScoping.DataAccess.Tests.Services; + +/// +/// Unit tests for ManualSyncRequestService. +/// Tests constructor validation, interface contract compliance, and static helper methods. +/// Note: Since this service uses Dapper with raw SQL, full integration tests with +/// an actual database are required for complete coverage of the SQL operations. +/// +public class ManualSyncRequestServiceTests +{ + private readonly IDbConnectionFactory _connectionFactory; + private readonly ILogger _logger; + + public ManualSyncRequestServiceTests() + { + _connectionFactory = Substitute.For(); + _logger = NullLogger.Instance; + } + + #region Constructor Tests + + [Fact] + public void Constructor_WithNullConnectionFactory_ThrowsArgumentNullException() + { + // Act & Assert + var exception = Should.Throw(() => + new ManualSyncRequestService(null!, _logger)); + + exception.ParamName.ShouldBe("connectionFactory"); + } + + [Fact] + public void Constructor_WithNullLogger_ThrowsArgumentNullException() + { + // Act & Assert + var exception = Should.Throw(() => + new ManualSyncRequestService(_connectionFactory, null!)); + + exception.ParamName.ShouldBe("logger"); + } + + [Fact] + public void Constructor_WithValidDependencies_CreatesInstance() + { + // Act + var service = new ManualSyncRequestService(_connectionFactory, _logger); + + // Assert + service.ShouldNotBeNull(); + } + + #endregion + + #region Interface Contract Tests + + [Fact] + public void ManualSyncRequestService_ImplementsIManualSyncRequestService() + { + // Arrange & Act + var service = new ManualSyncRequestService(_connectionFactory, _logger); + + // Assert + service.ShouldBeAssignableTo(); + } + + [Fact] + public void GetRequestsAsync_HasCorrectSignature() + { + // Arrange + var service = new ManualSyncRequestService(_connectionFactory, _logger); + + // Act - Verify method exists with correct return type + var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.GetRequestsAsync)); + + // Assert + methodInfo.ShouldNotBeNull(); + methodInfo.ReturnType.ShouldBe(typeof(Task>)); + } + + [Fact] + public void GetNextPendingRequestAsync_HasCorrectSignature() + { + // Arrange + var service = new ManualSyncRequestService(_connectionFactory, _logger); + + // Act - Verify method exists with correct return type + var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.GetNextPendingRequestAsync)); + + // Assert + methodInfo.ShouldNotBeNull(); + methodInfo.ReturnType.ShouldBe(typeof(Task)); + } + + [Fact] + public void CreateRequestAsync_HasCorrectSignature() + { + // Arrange + var service = new ManualSyncRequestService(_connectionFactory, _logger); + + // Act - Verify method exists with correct return type + var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.CreateRequestAsync)); + + // Assert + methodInfo.ShouldNotBeNull(); + methodInfo.ReturnType.ShouldBe(typeof(Task)); + } + + [Fact] + public void CancelRequestAsync_HasCorrectSignature() + { + // Arrange + var service = new ManualSyncRequestService(_connectionFactory, _logger); + + // Act - Verify method exists with correct return type + var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.CancelRequestAsync)); + + // Assert + methodInfo.ShouldNotBeNull(); + methodInfo.ReturnType.ShouldBe(typeof(Task)); + } + + [Fact] + public void CompleteRequestAsync_HasCorrectSignature() + { + // Arrange + var service = new ManualSyncRequestService(_connectionFactory, _logger); + + // Act - Verify method exists with correct return type + var methodInfo = typeof(ManualSyncRequestService).GetMethod(nameof(ManualSyncRequestService.CompleteRequestAsync)); + + // Assert + methodInfo.ShouldNotBeNull(); + methodInfo.ReturnType.ShouldBe(typeof(Task)); + } + + #endregion + + #region Domain Model Tests + + [Fact] + public void ManualSyncRequest_Status_WhenPending_ReturnsPending() + { + // Arrange + var request = new ManualSyncRequest + { + Id = 1, + PipelineName = "TestPipeline", + SyncType = "mass", + RequestedBy = "testuser", + RequestDT = DateTime.UtcNow, + CompletedDT = null, + CancelDT = null + }; + + // Act & Assert + request.Status.ShouldBe("Pending"); + } + + [Fact] + public void ManualSyncRequest_Status_WhenCompleted_ReturnsCompleted() + { + // Arrange + var request = new ManualSyncRequest + { + Id = 1, + PipelineName = "TestPipeline", + SyncType = "mass", + RequestedBy = "testuser", + RequestDT = DateTime.UtcNow.AddHours(-1), + CompletedDT = DateTime.UtcNow, + CancelDT = null + }; + + // Act & Assert + request.Status.ShouldBe("Completed"); + } + + [Fact] + public void ManualSyncRequest_Status_WhenCancelled_ReturnsCancelled() + { + // Arrange + var request = new ManualSyncRequest + { + Id = 1, + PipelineName = "TestPipeline", + SyncType = "mass", + RequestedBy = "testuser", + RequestDT = DateTime.UtcNow.AddHours(-1), + CompletedDT = null, + CancelDT = DateTime.UtcNow, + CancelledBy = "admin" + }; + + // Act & Assert + request.Status.ShouldBe("Cancelled"); + } + + [Fact] + public void ManualSyncRequest_Status_WhenCancelledAndCompleted_ReturnsCancelled() + { + // Arrange - Edge case: both CancelDT and CompletedDT are set + // Based on the implementation, CancelDT takes precedence + var request = new ManualSyncRequest + { + Id = 1, + PipelineName = "TestPipeline", + SyncType = "mass", + RequestedBy = "testuser", + RequestDT = DateTime.UtcNow.AddHours(-2), + CompletedDT = DateTime.UtcNow.AddHours(-1), + CancelDT = DateTime.UtcNow, + CancelledBy = "admin" + }; + + // Act & Assert + // CancelDT is checked first in the Status property, so it should return "Cancelled" + request.Status.ShouldBe("Cancelled"); + } + + [Fact] + public void ManualSyncRequest_DefaultRowVersion_IsEmptyArray() + { + // Arrange + var request = new ManualSyncRequest(); + + // Act & Assert + request.RowVersion.ShouldBeEmpty(); + } + + [Fact] + public void ManualSyncRequest_DefaultPipelineName_IsEmptyString() + { + // Arrange + var request = new ManualSyncRequest(); + + // Act & Assert + request.PipelineName.ShouldBe(string.Empty); + } + + [Fact] + public void ManualSyncRequest_DefaultSyncType_IsEmptyString() + { + // Arrange + var request = new ManualSyncRequest(); + + // Act & Assert + request.SyncType.ShouldBe(string.Empty); + } + + [Fact] + public void ManualSyncRequest_DefaultRequestedBy_IsEmptyString() + { + // Arrange + var request = new ManualSyncRequest(); + + // Act & Assert + request.RequestedBy.ShouldBe(string.Empty); + } + + [Fact] + public void ManualSyncRequest_CancelledBy_IsNullableAndDefaultsToNull() + { + // Arrange + var request = new ManualSyncRequest(); + + // Act & Assert + request.CancelledBy.ShouldBeNull(); + } + + #endregion + + #region Method Parameter Tests + + [Fact] + public void GetRequestsAsync_PendingOnlyParameter_DefaultsToFalse() + { + // Verify the interface defines correct default parameter + var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.GetRequestsAsync)); + var parameters = methodInfo!.GetParameters(); + + // The pendingOnly parameter should have a default value of false + var pendingOnlyParam = parameters.FirstOrDefault(p => p.Name == "pendingOnly"); + pendingOnlyParam.ShouldNotBeNull(); + pendingOnlyParam.HasDefaultValue.ShouldBeTrue(); + pendingOnlyParam.DefaultValue.ShouldBe(false); + } + + [Fact] + public void CreateRequestAsync_RequiresPipelineName() + { + // Verify the method has a pipelineName parameter + var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CreateRequestAsync)); + var parameters = methodInfo!.GetParameters(); + + var param = parameters.FirstOrDefault(p => p.Name == "pipelineName"); + param.ShouldNotBeNull(); + param.ParameterType.ShouldBe(typeof(string)); + } + + [Fact] + public void CreateRequestAsync_RequiresSyncType() + { + // Verify the method has a syncType parameter + var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CreateRequestAsync)); + var parameters = methodInfo!.GetParameters(); + + var param = parameters.FirstOrDefault(p => p.Name == "syncType"); + param.ShouldNotBeNull(); + param.ParameterType.ShouldBe(typeof(string)); + } + + [Fact] + public void CreateRequestAsync_RequiresRequestedBy() + { + // Verify the method has a requestedBy parameter + var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CreateRequestAsync)); + var parameters = methodInfo!.GetParameters(); + + var param = parameters.FirstOrDefault(p => p.Name == "requestedBy"); + param.ShouldNotBeNull(); + param.ParameterType.ShouldBe(typeof(string)); + } + + [Fact] + public void CancelRequestAsync_RequiresId() + { + // Verify the method has an id parameter + var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CancelRequestAsync)); + var parameters = methodInfo!.GetParameters(); + + var param = parameters.FirstOrDefault(p => p.Name == "id"); + param.ShouldNotBeNull(); + param.ParameterType.ShouldBe(typeof(int)); + } + + [Fact] + public void CancelRequestAsync_RequiresRowVersion() + { + // Verify the method has a rowVersion parameter for optimistic concurrency + var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CancelRequestAsync)); + var parameters = methodInfo!.GetParameters(); + + var param = parameters.FirstOrDefault(p => p.Name == "rowVersion"); + param.ShouldNotBeNull(); + param.ParameterType.ShouldBe(typeof(byte[])); + } + + [Fact] + public void CompleteRequestAsync_RequiresId() + { + // Verify the method has an id parameter + var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CompleteRequestAsync)); + var parameters = methodInfo!.GetParameters(); + + var param = parameters.FirstOrDefault(p => p.Name == "id"); + param.ShouldNotBeNull(); + param.ParameterType.ShouldBe(typeof(int)); + } + + [Fact] + public void CompleteRequestAsync_RequiresRowVersion() + { + // Verify the method has a rowVersion parameter for optimistic concurrency + var methodInfo = typeof(IManualSyncRequestService).GetMethod(nameof(IManualSyncRequestService.CompleteRequestAsync)); + var parameters = methodInfo!.GetParameters(); + + var param = parameters.FirstOrDefault(p => p.Name == "rowVersion"); + param.ShouldNotBeNull(); + param.ParameterType.ShouldBe(typeof(byte[])); + } + + #endregion +} diff --git a/NEW/tests/JdeScoping.DataSync.Tests/ScheduleCheckerTests.cs b/NEW/tests/JdeScoping.DataSync.Tests/ScheduleCheckerTests.cs index 28e0901..a3b278a 100644 --- a/NEW/tests/JdeScoping.DataSync.Tests/ScheduleCheckerTests.cs +++ b/NEW/tests/JdeScoping.DataSync.Tests/ScheduleCheckerTests.cs @@ -1,6 +1,7 @@ using JdeScoping.Core.Models; using JdeScoping.Core.Models.Enums; using JdeScoping.Core.Models.Infrastructure; +using JdeScoping.DataSync.Configuration; using JdeScoping.DataSync.Options; using JdeScoping.DataSync.Contracts; using JdeScoping.DataSync.Services; @@ -17,19 +18,28 @@ namespace JdeScoping.DataSync.Tests; public class ScheduleCheckerTests { private readonly IDataUpdateRepository _repository; + private readonly IPipelineRegistry _pipelineRegistry; private readonly IOptions _options; + private readonly List _pipelines; private readonly ScheduleChecker _sut; public ScheduleCheckerTests() { _repository = Substitute.For(); + _pipelineRegistry = Substitute.For(); + _pipelines = []; _options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions { LookbackMultiplier = 3, DataSources = [] }); + + // Setup pipeline registry to return our pipeline list + _pipelineRegistry.GetEnabledPipelines().Returns(_ => _pipelines); + _sut = new ScheduleChecker( _repository, + _pipelineRegistry, _options, NullLogger.Instance); } @@ -40,8 +50,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_WhenMassNeverRun_ReturnsMassTask() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", massEnabled: true, dailyEnabled: true, hourlyEnabled: true); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60); + _pipelines.Add(pipeline); _repository.GetLastDataUpdatesAsync(Arg.Any()) .Returns(new Dictionary()); @@ -59,10 +69,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_WhenMassDue_ReturnsMassOverDaily() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 60, - dailyEnabled: true, dailyInterval: 1440); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 60, dailyInterval: 1440); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-120), success: true); @@ -85,11 +93,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_WhenMassNotDue_ChecksDailyAndHourly() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, // weekly - dailyEnabled: true, dailyInterval: 1440, - hourlyEnabled: true, hourlyInterval: 60); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true); @@ -114,11 +119,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_WhenDailyDue_ReturnsDailyOverHourly() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440, - hourlyEnabled: true, hourlyInterval: 60); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true); @@ -145,11 +147,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_WhenOnlyHourlyDue_ReturnsHourly() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440, - hourlyEnabled: true, hourlyInterval: 60); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true); @@ -181,10 +180,8 @@ public class ScheduleCheckerTests { // Arrange: LookbackMultiplier = 3, daily interval = 1440 min // MinimumDT = lastDaily.EndDT - (3 * 1440) = lastDaily.EndDT - 4320 min = 3 days before lastDaily - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true); @@ -214,11 +211,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_HourlySync_UsesHourlyTimestampForMinimumDT() { // Arrange: Hourly uses its own timestamp and interval for MinimumDT calculation - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440, - hourlyEnabled: true, hourlyInterval: 60); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true); @@ -251,10 +245,8 @@ public class ScheduleCheckerTests { // Arrange: Test with multiplier = 5 _options.Value.LookbackMultiplier = 5; - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true); @@ -277,92 +269,15 @@ public class ScheduleCheckerTests #endregion - #region Disabled Table Handling + #region Manual-Only Pipelines [Fact] - public async Task GetPendingTasksAsync_DisabledDataSource_ReturnsNoTasks() + public async Task GetPendingTasksAsync_ManualOnlyPipeline_ReturnsNoTasks() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", massEnabled: true); - config.IsEnabled = false; - _options.Value.DataSources.Add(config); - - _repository.GetLastDataUpdatesAsync(Arg.Any()) - .Returns(new Dictionary()); - - // Act - var tasks = await _sut.GetPendingTasksAsync(); - - // Assert - tasks.ShouldBeEmpty(); - } - - [Fact] - public async Task GetPendingTasksAsync_DisabledMassSchedule_SkipsMass() - { - // Arrange: Mass disabled, Daily enabled - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: false, - dailyEnabled: true, dailyInterval: 1440); - _options.Value.DataSources.Add(config); - - var now = DateTime.UtcNow; - // Even with no mass ever run, if mass is disabled, should NOT require mass first - // However, current logic requires mass before daily, so this tests that properly - var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true); - - _repository.GetLastDataUpdatesAsync(Arg.Any()) - .Returns(new Dictionary - { - { "WorkOrder_3", lastMass } - }); - - // Act - var tasks = await _sut.GetPendingTasksAsync(); - - // Assert: Should return Daily since mass is disabled but already ran before - tasks.ShouldHaveSingleItem(); - tasks[0].UpdateType.ShouldBe(UpdateTypes.Daily); - } - - [Fact] - public async Task GetPendingTasksAsync_DisabledDailySchedule_SkipsDaily() - { - // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: false, - hourlyEnabled: true, hourlyInterval: 60); - _options.Value.DataSources.Add(config); - - var now = DateTime.UtcNow; - var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true); - var lastHourly = CreateDataUpdate("WorkOrder", UpdateTypes.Hourly, now.AddHours(-2), success: true); - - _repository.GetLastDataUpdatesAsync(Arg.Any()) - .Returns(new Dictionary - { - { "WorkOrder_3", lastMass }, - { "WorkOrder_1", lastHourly } - }); - - // Act - var tasks = await _sut.GetPendingTasksAsync(); - - // Assert: Should return Hourly, skipping Daily - tasks.ShouldHaveSingleItem(); - tasks[0].UpdateType.ShouldBe(UpdateTypes.Hourly); - } - - [Fact] - public async Task GetPendingTasksAsync_AllSchedulesDisabled_ReturnsNoTasks() - { - // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: false, - dailyEnabled: false, - hourlyEnabled: false); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080); + pipeline.IsManualOnly = true; + _pipelines.Add(pipeline); _repository.GetLastDataUpdatesAsync(Arg.Any()) .Returns(new Dictionary()); @@ -382,11 +297,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_NoPriorUpdates_RequiresMassFirst() { // Arrange: Never synced before, all schedules enabled - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440, - hourlyEnabled: true, hourlyInterval: 60); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60); + _pipelines.Add(pipeline); _repository.GetLastDataUpdatesAsync(Arg.Any()) .Returns(new Dictionary()); @@ -404,10 +316,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_OnlyMassCompleted_DailyHasNullMinimumDT() { // Arrange: Mass completed, no daily yet - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true); @@ -431,11 +341,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_NeverHadMass_DoesNotReturnDailyOrHourly() { // Arrange: Daily and Hourly enabled but no Mass ever run - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440, - hourlyEnabled: true, hourlyInterval: 60); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60); + _pipelines.Add(pipeline); _repository.GetLastDataUpdatesAsync(Arg.Any()) .Returns(new Dictionary()); @@ -456,9 +363,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_FailedMass_ReturnsMassAgain() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-5), success: false); @@ -481,10 +387,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_FailedDaily_ReturnsDailyAgain() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true); @@ -509,11 +413,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_FailedHourly_ReturnsHourlyAgain() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440, - hourlyEnabled: true, hourlyInterval: 60); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true); @@ -544,10 +445,10 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_MultipleTables_ReturnsTasksForEach() { // Arrange - var config1 = CreateDataSourceConfig("WorkOrder", massEnabled: true, massInterval: 60); - var config2 = CreateDataSourceConfig("LotUsage", massEnabled: true, massInterval: 60); - _options.Value.DataSources.Add(config1); - _options.Value.DataSources.Add(config2); + var pipeline1 = CreatePipeline("WorkOrder", massInterval: 60); + var pipeline2 = CreatePipeline("LotUsage", massInterval: 60); + _pipelines.Add(pipeline1); + _pipelines.Add(pipeline2); _repository.GetLastDataUpdatesAsync(Arg.Any()) .Returns(new Dictionary()); @@ -565,13 +466,10 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_MultipleTables_DifferentSchedulesDue() { // Arrange - var config1 = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440); - var config2 = CreateDataSourceConfig("LotUsage", - massEnabled: true, massInterval: 60); - _options.Value.DataSources.Add(config1); - _options.Value.DataSources.Add(config2); + var pipeline1 = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440); + var pipeline2 = CreatePipeline("LotUsage", massInterval: 60); + _pipelines.Add(pipeline1); + _pipelines.Add(pipeline2); var now = DateTime.UtcNow; var lastMassWorkOrder = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true); @@ -603,11 +501,8 @@ public class ScheduleCheckerTests public async Task GetPendingTasksAsync_NothingDue_ReturnsEmptyList() { // Arrange - var config = CreateDataSourceConfig("WorkOrder", - massEnabled: true, massInterval: 10080, - dailyEnabled: true, dailyInterval: 1440, - hourlyEnabled: true, hourlyInterval: 60); - _options.Value.DataSources.Add(config); + var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60); + _pipelines.Add(pipeline); var now = DateTime.UtcNow; // All syncs completed recently @@ -631,9 +526,9 @@ public class ScheduleCheckerTests } [Fact] - public async Task GetPendingTasksAsync_NoDataSources_ReturnsEmptyList() + public async Task GetPendingTasksAsync_NoPipelines_ReturnsEmptyList() { - // Arrange: No data sources configured + // Arrange: No pipelines configured _repository.GetLastDataUpdatesAsync(Arg.Any()) .Returns(new Dictionary()); @@ -648,35 +543,29 @@ public class ScheduleCheckerTests #region Helper Methods - private static DataSourceConfig CreateDataSourceConfig( - string tableName, - bool massEnabled = false, - int massInterval = 10080, - bool dailyEnabled = false, - int dailyInterval = 1440, - bool hourlyEnabled = false, - int hourlyInterval = 60) + private static EtlPipelineConfig CreatePipeline( + string name, + int? massInterval = null, + int? dailyInterval = null, + int? hourlyInterval = null) { - return new DataSourceConfig + return new EtlPipelineConfig { - TableName = tableName, - SourceSystem = "JDE", - SourceData = tableName.ToUpper(), + Name = name, IsEnabled = true, - MassConfig = new ScheduleConfig + IsManualOnly = false, + MassSyncIntervalMinutes = massInterval, + DailySyncIntervalMinutes = dailyInterval, + HourlySyncIntervalMinutes = hourlyInterval, + Source = new SourceElement { - Enabled = massEnabled, - IntervalMinutes = massInterval + Connection = "jde", + Query = "SELECT * FROM Test" }, - DailyConfig = new ScheduleConfig + Destination = new DestinationElement { - Enabled = dailyEnabled, - IntervalMinutes = dailyInterval - }, - HourlyConfig = new ScheduleConfig - { - Enabled = hourlyEnabled, - IntervalMinutes = hourlyInterval + Table = name, + MatchColumns = ["Id"] } }; } diff --git a/NEW/tests/JdeScoping.DataSync.Tests/Services/PipelineRegistryTests.cs b/NEW/tests/JdeScoping.DataSync.Tests/Services/PipelineRegistryTests.cs new file mode 100644 index 0000000..3cf6061 --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Tests/Services/PipelineRegistryTests.cs @@ -0,0 +1,434 @@ +using JdeScoping.DataSync.Configuration; +using JdeScoping.DataSync.Options; +using JdeScoping.DataSync.Services; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using Shouldly; +using System.Text.Json; +using Xunit; +using MsOptions = Microsoft.Extensions.Options.Options; + +namespace JdeScoping.DataSync.Tests.Services; + +public class PipelineRegistryTests : IDisposable +{ + private readonly string _testDirectory; + private readonly IPipelineValidator _validator; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly IHostEnvironment _environment; + + public PipelineRegistryTests() + { + _testDirectory = Path.Combine(Path.GetTempPath(), $"PipelineRegistryTests_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_testDirectory); + + _validator = new PipelineValidator(); + _logger = Substitute.For>(); + _environment = Substitute.For(); + _environment.ContentRootPath.Returns(_testDirectory); + + _options = MsOptions.Create(new DataSyncOptions + { + PipelinesDirectory = "Pipelines", + StrictPipelineValidation = false + }); + + // Create the Pipelines subdirectory + Directory.CreateDirectory(Path.Combine(_testDirectory, "Pipelines")); + } + + public void Dispose() + { + if (Directory.Exists(_testDirectory)) + { + Directory.Delete(_testDirectory, recursive: true); + } + } + + private string PipelinesDir => Path.Combine(_testDirectory, "Pipelines"); + + #region Loading Tests + + [Fact] + public async Task LoadPipelines_ValidDirectory_LoadsAll() + { + // Arrange + CreatePipelineFile("Pipeline1", true); + CreatePipelineFile("Pipeline2", true); + CreatePipelineFile("Pipeline3", true); + + var registry = CreateRegistry(); + + // Act + var result = await registry.ReloadAsync(); + + // Assert + result.Success.ShouldBeTrue(); + result.PipelinesLoaded.ShouldBe(3); + registry.GetAllPipelines().Count.ShouldBe(3); + } + + [Fact] + public async Task LoadPipelines_EmptyDirectory_ReturnsEmpty() + { + // Arrange - empty directory (Pipelines subdirectory is already created but empty) + var registry = CreateRegistry(); + + // Act + var result = await registry.ReloadAsync(); + + // Assert + result.Success.ShouldBeTrue(); + result.PipelinesLoaded.ShouldBe(0); + registry.GetAllPipelines().ShouldBeEmpty(); + } + + [Fact] + public async Task LoadPipelines_OnlyLoadsJsonFiles() + { + // Arrange + CreatePipelineFile("ValidPipeline", true); + File.WriteAllText(Path.Combine(PipelinesDir, "readme.txt"), "Some text"); + File.WriteAllText(Path.Combine(PipelinesDir, "config.xml"), ""); + + var registry = CreateRegistry(); + + // Act + var result = await registry.ReloadAsync(); + + // Assert + result.PipelinesLoaded.ShouldBe(1); + } + + #endregion + + #region Error Handling Tests + + [Fact] + public async Task LoadPipelines_InvalidJson_ReturnsError() + { + // Arrange + File.WriteAllText(Path.Combine(PipelinesDir, "pipeline.Invalid.json"), "{ invalid json }"); + + var registry = CreateRegistry(); + + // Act + var result = await registry.ReloadAsync(); + + // Assert + result.Errors.ShouldContain(e => e.ErrorType.Contains("parse", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task LoadPipelines_DuplicateNames_ReturnsError() + { + // Arrange - two files with same pipeline name + CreatePipelineFile("DuplicateName", true, "pipeline.First.json"); + CreatePipelineFile("DuplicateName", true, "pipeline.Second.json"); + + var registry = CreateRegistry(); + + // Act + var result = await registry.ReloadAsync(); + + // Assert + result.Errors.ShouldContain(e => e.ErrorType.Contains("validation", StringComparison.OrdinalIgnoreCase) + && e.Messages.Any(m => m.Contains("Duplicate", StringComparison.OrdinalIgnoreCase))); + } + + #endregion + + #region Retrieval Tests + + [Fact] + public async Task GetPipeline_ByName_ReturnsCorrect() + { + // Arrange + CreatePipelineFile("TestPipeline", true); + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act + var pipeline = registry.GetPipeline("TestPipeline"); + + // Assert + pipeline.ShouldNotBeNull(); + pipeline.Name.ShouldBe("TestPipeline"); + } + + [Fact] + public async Task GetPipeline_CaseInsensitive_ReturnsCorrect() + { + // Arrange + CreatePipelineFile("MyPipeline", true); + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act + var pipeline = registry.GetPipeline("mypipeline"); + + // Assert + pipeline.ShouldNotBeNull(); + pipeline.Name.ShouldBe("MyPipeline"); + } + + [Fact] + public async Task GetPipeline_NotFound_ReturnsNull() + { + // Arrange + CreatePipelineFile("ExistingPipeline", true); + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act + var pipeline = registry.GetPipeline("NonExistentPipeline"); + + // Assert + pipeline.ShouldBeNull(); + } + + [Fact] + public async Task GetEnabledPipelines_OnlyReturnsEnabled() + { + // Arrange + CreatePipelineFile("EnabledPipeline1", true); + CreatePipelineFile("EnabledPipeline2", true); + CreatePipelineFile("DisabledPipeline", false); + + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act + var enabledPipelines = registry.GetEnabledPipelines(); + + // Assert + enabledPipelines.Count.ShouldBe(2); + enabledPipelines.ShouldAllBe(p => p.IsEnabled); + } + + [Fact] + public async Task GetAllPipelines_IncludesDisabled() + { + // Arrange + CreatePipelineFile("EnabledPipeline", true); + CreatePipelineFile("DisabledPipeline", false); + + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act + var allPipelines = registry.GetAllPipelines(); + + // Assert + allPipelines.Count.ShouldBe(2); + allPipelines.ShouldContain(p => !p.IsEnabled); + } + + #endregion + + #region Reload Tests + + [Fact] + public async Task ReloadAsync_UpdatesSnapshot() + { + // Arrange + CreatePipelineFile("Pipeline1", true); + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Add another pipeline + CreatePipelineFile("Pipeline2", true); + + // Act + await registry.ReloadAsync(); + + // Assert + registry.GetAllPipelines().Count.ShouldBe(2); + } + + [Fact] + public async Task ReloadAsync_IncrementsVersion() + { + // Arrange + CreatePipelineFile("Pipeline1", true); + var registry = CreateRegistry(); + + // Act + await registry.ReloadAsync(); + var version1 = registry.Version; + + await registry.ReloadAsync(); + var version2 = registry.Version; + + // Assert + version2.ShouldBe(version1 + 1); + } + + [Fact] + public async Task ReloadAsync_UpdatesLastLoadedAt() + { + // Arrange + CreatePipelineFile("Pipeline1", true); + var registry = CreateRegistry(); + + // Act + var before = DateTime.UtcNow; + await registry.ReloadAsync(); + var after = DateTime.UtcNow; + + // Assert + registry.LastLoadedAt.ShouldNotBeNull(); + registry.LastLoadedAt.Value.ShouldBeGreaterThanOrEqualTo(before.AddSeconds(-1)); + registry.LastLoadedAt.Value.ShouldBeLessThanOrEqualTo(after.AddSeconds(1)); + } + + #endregion + + #region IsValidPipelineAndSyncType Tests + + [Fact] + public async Task IsValidPipelineAndSyncType_ValidCombination_ReturnsTrue() + { + // Arrange + CreatePipelineFile("TestPipeline", true, massSyncInterval: 1440, dailySyncInterval: 60); + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act & Assert + registry.IsValidPipelineAndSyncType("TestPipeline", "mass").ShouldBeTrue(); + registry.IsValidPipelineAndSyncType("TestPipeline", "daily").ShouldBeTrue(); + } + + [Fact] + public async Task IsValidPipelineAndSyncType_UnsupportedSyncType_ReturnsFalse() + { + // Arrange - only mass sync + CreatePipelineFile("TestPipeline", true, massSyncInterval: 1440); + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act & Assert + registry.IsValidPipelineAndSyncType("TestPipeline", "hourly").ShouldBeFalse(); + } + + [Fact] + public async Task IsValidPipelineAndSyncType_UnknownPipeline_ReturnsFalse() + { + // Arrange + CreatePipelineFile("TestPipeline", true); + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act & Assert + registry.IsValidPipelineAndSyncType("UnknownPipeline", "mass").ShouldBeFalse(); + } + + [Fact] + public async Task IsValidPipelineAndSyncType_DisabledPipeline_ReturnsFalse() + { + // Arrange + CreatePipelineFile("DisabledPipeline", false, massSyncInterval: 1440); + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act & Assert + registry.IsValidPipelineAndSyncType("DisabledPipeline", "mass").ShouldBeFalse(); + } + + #endregion + + #region Thread Safety Tests + + [Fact] + public async Task ReadOperations_ThreadSafe() + { + // Arrange + for (int i = 0; i < 10; i++) + { + CreatePipelineFile($"Pipeline{i}", true); + } + + var registry = CreateRegistry(); + await registry.ReloadAsync(); + + // Act - multiple concurrent reads + var tasks = Enumerable.Range(0, 100).Select(_ => Task.Run(() => + { + var all = registry.GetAllPipelines(); + var enabled = registry.GetEnabledPipelines(); + var specific = registry.GetPipeline("Pipeline5"); + return all.Count + enabled.Count + (specific != null ? 1 : 0); + })); + + // Assert - no exceptions + var results = await Task.WhenAll(tasks); + results.ShouldAllBe(r => r > 0); + } + + [Fact] + public async Task ConcurrentReloads_Serialized() + { + // Arrange + CreatePipelineFile("Pipeline1", true); + var registry = CreateRegistry(); + + // Act - multiple concurrent reloads + var tasks = Enumerable.Range(0, 10).Select(_ => registry.ReloadAsync()); + + // Assert - no exceptions and final state is valid + await Task.WhenAll(tasks); + registry.GetAllPipelines().Count.ShouldBe(1); + } + + #endregion + + #region Helper Methods + + private PipelineRegistry CreateRegistry() => + new PipelineRegistry(_options, _validator, _logger, _environment); + + private void CreatePipelineFile( + string name, + bool isEnabled, + string? fileName = null, + int? massSyncInterval = 1440, + int? dailySyncInterval = null, + int? hourlySyncInterval = null) + { + var pipeline = new + { + name, + isEnabled, + isManualOnly = !massSyncInterval.HasValue && !dailySyncInterval.HasValue && !hourlySyncInterval.HasValue && !isEnabled, + massSyncIntervalMinutes = massSyncInterval, + dailySyncIntervalMinutes = dailySyncInterval, + hourlySyncIntervalMinutes = hourlySyncInterval, + source = new + { + connection = "jde", + query = "SELECT * FROM TestTable" + }, + destination = new + { + table = $"{name}_Table", + matchColumns = new[] { "Id" } + } + }; + + var json = JsonSerializer.Serialize(pipeline, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }); + + var actualFileName = fileName ?? $"pipeline.{name}.json"; + var filePath = Path.Combine(PipelinesDir, actualFileName); + File.WriteAllText(filePath, json); + } + + #endregion +} diff --git a/NEW/tests/JdeScoping.DataSync.Tests/Services/PipelineValidatorTests.cs b/NEW/tests/JdeScoping.DataSync.Tests/Services/PipelineValidatorTests.cs new file mode 100644 index 0000000..51ff658 --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Tests/Services/PipelineValidatorTests.cs @@ -0,0 +1,483 @@ +using JdeScoping.DataSync.Configuration; +using JdeScoping.DataSync.Services; +using Shouldly; +using Xunit; + +namespace JdeScoping.DataSync.Tests.Services; + +public class PipelineValidatorTests +{ + private readonly IPipelineValidator _validator; + + public PipelineValidatorTests() + { + _validator = new PipelineValidator(); + } + + #region Name/Filename Matching + + [Fact] + public void Validate_NameMatchesFilename_Passes() + { + // Arrange + var pipeline = CreateValidPipeline("TestPipeline"); + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeTrue(); + result.Errors.ShouldBeEmpty(); + } + + [Fact] + public void Validate_NameMismatchFilename_Fails() + { + // Arrange + var pipeline = CreateValidPipeline("WrongName"); + + // Act + var result = _validator.Validate(pipeline, "pipeline.CorrectName.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("does not match filename", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_NameCaseInsensitive_Passes() + { + // Arrange + var pipeline = CreateValidPipeline("testpipeline"); + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeTrue(); + } + + #endregion + + #region Source Validation + + [Fact] + public void Validate_MissingSource_Fails() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + MassSyncIntervalMinutes = 1440, + Source = null!, + Destination = CreateValidDestination() + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("Source is required", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_MissingSourceConnection_Fails() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + MassSyncIntervalMinutes = 1440, + Source = new SourceElement + { + Connection = "", + Query = "SELECT * FROM table" + }, + Destination = CreateValidDestination() + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("Connection is required", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_InvalidConnection_Fails() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + MassSyncIntervalMinutes = 1440, + Source = new SourceElement + { + Connection = "invalid_db", + Query = "SELECT * FROM table" + }, + Destination = CreateValidDestination() + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("not valid", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_ValidConnections_Pass() + { + // Arrange & Act & Assert + foreach (var connection in new[] { "jde", "cms", "giw", "lotfinder" }) + { + var pipeline = CreateValidPipeline("TestPipeline"); + pipeline.Source.Connection = connection; + + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + result.IsValid.ShouldBeTrue($"Connection '{connection}' should be valid"); + } + } + + [Fact] + public void Validate_MissingSourceQuery_Fails() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + MassSyncIntervalMinutes = 1440, + Source = new SourceElement + { + Connection = "jde", + Query = "" + }, + Destination = CreateValidDestination() + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("Query is required", StringComparison.OrdinalIgnoreCase)); + } + + #endregion + + #region Destination Validation + + [Fact] + public void Validate_MissingDestination_Fails() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + MassSyncIntervalMinutes = 1440, + Source = CreateValidSource(), + Destination = null! + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("Destination is required", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_MissingDestinationTable_Fails() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + MassSyncIntervalMinutes = 1440, + Source = CreateValidSource(), + Destination = new DestinationElement + { + Table = "", + MatchColumns = ["Id"] + } + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("Table is required", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_EmptyMatchColumns_Fails() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + MassSyncIntervalMinutes = 1440, + Source = CreateValidSource(), + Destination = new DestinationElement + { + Table = "TestTable", + MatchColumns = [] + } + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("MatchColumns", StringComparison.OrdinalIgnoreCase)); + } + + #endregion + + #region Interval Validation + + [Fact] + public void Validate_EnabledWithoutAnyInterval_Fails() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + IsManualOnly = false, + // No intervals set + Source = CreateValidSource(), + Destination = CreateValidDestination() + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("At least one sync interval", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_ManualOnlyWithoutInterval_Passes() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + IsManualOnly = true, + // No intervals set - ok for manual-only + Source = CreateValidSource(), + Destination = CreateValidDestination() + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeTrue(); + } + + [Fact] + public void Validate_DisabledWithoutInterval_Passes() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = false, + // No intervals set - ok for disabled + Source = CreateValidSource(), + Destination = CreateValidDestination() + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeTrue(); + } + + [Fact] + public void Validate_ZeroMassInterval_Fails() + { + // Arrange + var pipeline = CreateValidPipeline("TestPipeline"); + pipeline.MassSyncIntervalMinutes = 0; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("MassSyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_NegativeInterval_Fails() + { + // Arrange + var pipeline = CreateValidPipeline("TestPipeline"); + pipeline.DailySyncIntervalMinutes = -60; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("DailySyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase)); + } + + #endregion + + #region Warning Cases + + [Fact] + public void Validate_HourlyWithoutDaily_ReturnsWarning() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "TestPipeline", + IsEnabled = true, + HourlySyncIntervalMinutes = 15, + // No daily interval + Source = CreateValidSource(), + Destination = CreateValidDestination() + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeTrue(); // Warnings don't fail validation + result.Warnings.ShouldContain(w => w.Contains("daily", StringComparison.OrdinalIgnoreCase)); + } + + #endregion + + #region Script Validation + + [Fact] + public void Validate_PreScriptWithEmptyScript_Fails() + { + // Arrange + var pipeline = CreateValidPipeline("TestPipeline"); + pipeline.PreScripts = [new ScriptElement { Script = "" }]; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("PreScripts", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_PostScriptWithEmptyScript_Fails() + { + // Arrange + var pipeline = CreateValidPipeline("TestPipeline"); + pipeline.PostScripts = [new ScriptElement { Script = "" }]; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => e.Contains("PostScripts", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_ValidScripts_Passes() + { + // Arrange + var pipeline = CreateValidPipeline("TestPipeline"); + pipeline.PreScripts = [new ScriptElement { Script = "TRUNCATE TABLE Staging" }]; + pipeline.PostScripts = [new ScriptElement { Script = "EXEC ProcessData" }]; + + // Act + var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json"); + + // Assert + result.IsValid.ShouldBeTrue(); + } + + #endregion + + #region Complete Valid Pipeline + + [Fact] + public void Validate_CompleteValidPipeline_Passes() + { + // Arrange + var pipeline = new EtlPipelineConfig + { + Name = "WorkOrder_Curr", + IsEnabled = true, + MassSyncIntervalMinutes = 1440, + DailySyncIntervalMinutes = 60, + HourlySyncIntervalMinutes = 15, + Source = new SourceElement + { + Connection = "jde", + Query = "SELECT * FROM WorkOrders WHERE ModDate > @lastSync", + MassQuery = "SELECT * FROM WorkOrders" + }, + Destination = new DestinationElement + { + Table = "WorkOrder_Curr", + MatchColumns = ["OrderNumber"] + } + }; + + // Act + var result = _validator.Validate(pipeline, "pipeline.WorkOrder_Curr.json"); + + // Assert + result.IsValid.ShouldBeTrue(); + result.Errors.ShouldBeEmpty(); + } + + #endregion + + #region Helper Methods + + private static EtlPipelineConfig CreateValidPipeline(string name) => new() + { + Name = name, + IsEnabled = true, + MassSyncIntervalMinutes = 1440, + Source = CreateValidSource(), + Destination = CreateValidDestination() + }; + + private static SourceElement CreateValidSource() => new() + { + Connection = "jde", + Query = "SELECT * FROM TestTable" + }; + + private static DestinationElement CreateValidDestination() => new() + { + Table = "TestTable", + MatchColumns = ["Id"] + }; + + #endregion +}