diff --git a/.gitignore b/.gitignore index b5daf08..5a15c77 100644 --- a/.gitignore +++ b/.gitignore @@ -427,3 +427,16 @@ secrets.json # Compressed cache files *.zstd + +# SecureStore key files +*.secrets.key + +# Temporary documentation drafts +JdeScoping-src-docs-*.md + +# Credential notes +credentials_info.md + +# Ad-hoc SQL and reports +windchill_query.sql +mis_report.md diff --git a/CLAUDE.md b/CLAUDE.md index 1a46496..37911b1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,11 +11,25 @@ This is a **migration project** to convert a legacy .NET Framework 4.8 applicati JdeScopingTool/ ├── OLD/ # Legacy .NET Framework 4.8 source code (read-only reference) ├── NEW/ # New .NET 10 solution (build here) +│ └── JdeScoping.slnx # Solution file (.slnx format) ├── openspec/ # OpenSpec specifications and design documents ├── PLANS/ # Design plans, implementation plans, and task lists └── DOCUMENTATION/ # Project documentation ``` +### Build Commands +The solution uses the `.slnx` format (XML-based solution file): +```bash +# Build entire solution +dotnet build NEW/JdeScoping.slnx + +# Run all tests +dotnet test NEW/JdeScoping.slnx + +# Build specific project +dotnet build NEW/src/JdeScoping.Host/JdeScoping.Host.csproj +``` + ### Legacy System Purpose A manufacturing/ERP search tool that: - Caches data from JDE (JD Edwards - Oracle) and CMS (Sybase) enterprise systems into SQL Server diff --git a/DOCUMENTATION/Architecture/Configuration.md b/DOCUMENTATION/Architecture/Configuration.md index b4cd379..31484e1 100644 --- a/DOCUMENTATION/Architecture/Configuration.md +++ b/DOCUMENTATION/Architecture/Configuration.md @@ -105,6 +105,86 @@ builder.Services.Configure(builder.Configuration.GetSection(" builder.Services.Configure(builder.Configuration.GetSection("Auth")); ``` +## Configuration Validation + +The application validates configuration at startup using `ConfigurationValidationRunner`. This catches configuration errors early, before the application attempts to use misconfigured services. + +### How validation works + +1. The runner resolves all `IConfigurationValidator` implementations from DI +2. Validators execute in order (sorted by `Order` property, lower values run first) +3. Each validator returns a `ConfigurationValidationResult` with errors and warnings +4. Warnings are logged but don't prevent startup +5. Errors cause startup to fail with detailed logging + +### Built-in validators + +| Validator | Order | Purpose | +|-----------|-------|---------| +| `SecureStoreValidator` | 100 | Validates required secrets exist in the SecureStore | +| `LdapOptionsValidator` | 200 | Validates LDAP configuration settings | + +### Creating a new validator + +Implement `IConfigurationValidator` in `JdeScoping.Infrastructure/Validation/`: + +```csharp +using JdeScoping.Core.Validation; + +public class MyOptionsValidator : IConfigurationValidator +{ + private readonly MyOptions _options; + + public int Order => 300; // Run after existing validators + public string Name => "MyOptions"; + + public MyOptionsValidator(IOptions options) + { + _options = options.Value; + } + + public ConfigurationValidationResult Validate() + { + var result = new ConfigurationValidationResult(Name); + + if (string.IsNullOrEmpty(_options.RequiredSetting)) + result.AddError("RequiredSetting must be configured"); + + if (_options.Timeout < 1000) + result.AddWarning("Timeout below 1000ms may cause issues"); + + return result; + } +} +``` + +Register the validator in `DependencyInjection.cs`: + +```csharp +services.AddSingleton(); +``` + +The runner automatically discovers and executes all registered validators. + +### Validation result types + +- **Errors** - Fatal configuration problems that prevent startup. Use `result.AddError()` for missing required settings, invalid values, or conditions that would cause runtime failures. +- **Warnings** - Non-fatal issues that should be logged but allow startup. Use `result.AddWarning()` for suboptimal settings or deprecated configurations. + +### Order conventions + +Follow these conventions for the `Order` property: + +| Range | Purpose | +|-------|---------| +| 100 | Security/secrets (SecureStore) | +| 200 | Authentication (LDAP) | +| 300-399 | Data sources and connections | +| 400-499 | Service-specific validation | +| 500+ | Application-level validation | + +Lower-ordered validators run first, allowing dependent validators to assume earlier validations passed. + ## Data Source Configuration The JDE and CMS data sources support two implementations: diff --git a/NEW/src/JdeScoping.Core/Models/Infrastructure/DataUpdate.cs b/NEW/src/JdeScoping.Core/Models/Infrastructure/DataUpdate.cs index 85c24eb..0fdc941 100644 --- a/NEW/src/JdeScoping.Core/Models/Infrastructure/DataUpdate.cs +++ b/NEW/src/JdeScoping.Core/Models/Infrastructure/DataUpdate.cs @@ -51,4 +51,9 @@ public class DataUpdate /// Number of records in update /// public long NumberRecords { get; set; } + + /// + /// JSON string of parameters used during the sync operation (key:value pairs). + /// + public string? Parameters { get; set; } } diff --git a/NEW/src/JdeScoping.Core/Options/SecureStoreOptions.cs b/NEW/src/JdeScoping.Core/Options/SecureStoreOptions.cs index b8db4c6..ed11905 100644 --- a/NEW/src/JdeScoping.Core/Options/SecureStoreOptions.cs +++ b/NEW/src/JdeScoping.Core/Options/SecureStoreOptions.cs @@ -34,7 +34,7 @@ public class SecureStoreOptions public bool AutoCreateStore { get; set; } = true; /// - /// Whether to migrate existing secrets (RSA key, Excel passwords) on startup. + /// List of secret keys that must exist in the store for the application to start. /// - public bool MigrateExistingSecrets { get; set; } = true; + public List RequiredKeys { get; set; } = []; } diff --git a/NEW/src/JdeScoping.Core/Validation/ConfigurationValidationResult.cs b/NEW/src/JdeScoping.Core/Validation/ConfigurationValidationResult.cs new file mode 100644 index 0000000..d2562a1 --- /dev/null +++ b/NEW/src/JdeScoping.Core/Validation/ConfigurationValidationResult.cs @@ -0,0 +1,45 @@ +namespace JdeScoping.Core.Validation; + +/// +/// Result of a configuration validation check. +/// +public class ConfigurationValidationResult +{ + /// + /// Name of the validator that produced this result. + /// + public string ValidatorName { get; } + + /// + /// True if validation passed (no errors). + /// + public bool IsValid => Errors.Count == 0; + + /// + /// List of validation errors that prevent startup. + /// + public List Errors { get; } = []; + + /// + /// List of validation warnings (non-fatal). + /// + public List Warnings { get; } = []; + + /// + /// Creates a new validation result for a named validator. + /// + /// Name of the validator. + public ConfigurationValidationResult(string validatorName) => ValidatorName = validatorName; + + /// + /// Adds an error message to the result. + /// + /// The error message. + public void AddError(string message) => Errors.Add(message); + + /// + /// Adds a warning message to the result. + /// + /// The warning message. + public void AddWarning(string message) => Warnings.Add(message); +} diff --git a/NEW/src/JdeScoping.Core/Validation/IConfigurationValidator.cs b/NEW/src/JdeScoping.Core/Validation/IConfigurationValidator.cs new file mode 100644 index 0000000..06c51d3 --- /dev/null +++ b/NEW/src/JdeScoping.Core/Validation/IConfigurationValidator.cs @@ -0,0 +1,24 @@ +namespace JdeScoping.Core.Validation; + +/// +/// Interface for configuration validators that run on application startup. +/// +public interface IConfigurationValidator +{ + /// + /// Order in which this validator runs. Lower values run first. + /// Convention: 100=SecureStore, 200=LDAP, 300+=future validators. + /// + int Order { get; } + + /// + /// Display name for logging purposes. + /// + string Name { get; } + + /// + /// Validates the configuration and returns a result with any errors or warnings. + /// + /// Validation result containing errors and warnings. + ConfigurationValidationResult Validate(); +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/Pipelines/dev-pipelines.json b/NEW/src/JdeScoping.DataSync.Dev/Pipelines/dev-pipelines.json index 751e24e..6cb4ae9 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/Pipelines/dev-pipelines.json +++ b/NEW/src/JdeScoping.DataSync.Dev/Pipelines/dev-pipelines.json @@ -2,8 +2,8 @@ "settings": { "sizeCategories": { "small": ["Branch", "OrgHierarchy", "WorkCenter", "ProfitCenter"], - "medium": ["JdeUser", "FunctionCode", "Item", "RouteMaster", "MisData_Curr"], - "large": ["Lot", "MisData_Hist", "WorkOrder_Curr", "WorkOrder_Hist", "LotUsage_Hist", "WorkOrderComponent_Hist"], + "medium": ["JdeUser", "FunctionCode", "Item", "RouteMaster"], + "large": ["Lot", "MisData_Curr", "MisData_Hist", "WorkOrder_Curr", "WorkOrder_Hist", "LotUsage_Hist", "WorkOrderComponent_Hist"], "veryLarge": ["WorkOrderStep_Hist", "WorkOrderComponent_Curr", "WorkOrderRouting", "LotUsage_Curr", "WorkOrderStep_Curr", "WorkOrderTime_Hist", "WorkOrderTime_Curr"] } }, diff --git a/NEW/src/JdeScoping.DataSync/Contracts/IDataUpdateRepository.cs b/NEW/src/JdeScoping.DataSync/Contracts/IDataUpdateRepository.cs index c8c1b33..afd9065 100644 --- a/NEW/src/JdeScoping.DataSync/Contracts/IDataUpdateRepository.cs +++ b/NEW/src/JdeScoping.DataSync/Contracts/IDataUpdateRepository.cs @@ -23,6 +23,7 @@ public interface IDataUpdateRepository /// Source data identifier. /// Target table name. /// Type of update. + /// Optional JSON string of parameters used during the sync operation. /// Cancellation token. /// The ID of the created record. Task StartUpdateAsync( @@ -30,6 +31,7 @@ public interface IDataUpdateRepository string sourceData, string tableName, UpdateTypes updateType, + string? parameters = null, CancellationToken cancellationToken = default); /// diff --git a/NEW/src/JdeScoping.DataSync/Services/DataUpdateRepository.cs b/NEW/src/JdeScoping.DataSync/Services/DataUpdateRepository.cs index 0dea1bc..a4f0516 100644 --- a/NEW/src/JdeScoping.DataSync/Services/DataUpdateRepository.cs +++ b/NEW/src/JdeScoping.DataSync/Services/DataUpdateRepository.cs @@ -50,7 +50,8 @@ SELECT cte.ID, cte.EndDT, cte.UpdateType, cte.WasSuccessful, - cte.NumberRecords + cte.NumberRecords, + cte.Parameters FROM DU_CTE cte WHERE cte.RN = 1"; @@ -68,12 +69,13 @@ WHERE cte.RN = 1"; string sourceData, string tableName, UpdateTypes updateType, + string? parameters = null, CancellationToken cancellationToken = default) { const string sql = @" -INSERT INTO dbo.DataUpdate (SourceSystem, SourceData, TableName, UpdateType, StartDT, NumberRecords, WasSuccessful) +INSERT INTO dbo.DataUpdate (SourceSystem, SourceData, TableName, UpdateType, StartDT, NumberRecords, WasSuccessful, Parameters) OUTPUT INSERTED.ID -VALUES (@sourceSystem, @sourceData, @tableName, @updateType, GETUTCDATE(), @inProgressMarker, 0)"; +VALUES (@sourceSystem, @sourceData, @tableName, @updateType, GETUTCDATE(), @inProgressMarker, 0, @parameters)"; await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(cancellationToken); var id = await connection.ExecuteScalarAsync( @@ -84,7 +86,8 @@ VALUES (@sourceSystem, @sourceData, @tableName, @updateType, GETUTCDATE(), @inPr sourceData, tableName, updateType = (int)updateType, - inProgressMarker = InProgressMarker + inProgressMarker = InProgressMarker, + parameters }, commandTimeout: 30); @@ -268,12 +271,12 @@ FROM LastSuccessful"; { var sql = updateType.HasValue ? @"SELECT TOP (@count) du.Id, du.SourceSystem, du.SourceData, du.TableName, - du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords + du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords, du.Parameters FROM dbo.DataUpdate du WHERE du.TableName = @tableName AND du.UpdateType = @updateType ORDER BY du.StartDt DESC" : @"SELECT TOP (@count) du.Id, du.SourceSystem, du.SourceData, du.TableName, - du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords + du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords, du.Parameters FROM dbo.DataUpdate du WHERE du.TableName = @tableName ORDER BY du.StartDt DESC"; @@ -299,7 +302,7 @@ WITH LastRuns AS ( FROM dbo.DataUpdate du WHERE du.TableName = @tableName ) -SELECT Id, SourceSystem, SourceData, TableName, StartDt, EndDt, UpdateType, WasSuccessful, NumberRecords +SELECT Id, SourceSystem, SourceData, TableName, StartDt, EndDt, UpdateType, WasSuccessful, NumberRecords, Parameters FROM LastRuns WHERE RN = 1"; diff --git a/NEW/src/JdeScoping.DataSync/Services/TableSyncOperation.cs b/NEW/src/JdeScoping.DataSync/Services/TableSyncOperation.cs index 3b16576..675e4eb 100644 --- a/NEW/src/JdeScoping.DataSync/Services/TableSyncOperation.cs +++ b/NEW/src/JdeScoping.DataSync/Services/TableSyncOperation.cs @@ -64,12 +64,16 @@ public class TableSyncOperation : ITableSyncOperation _metrics.RecordOperationStarted(task.TableName, task.UpdateType.ToString()); + // Build parameters JSON for tracking + var parametersJson = BuildParametersJson(task); + // Log start of data update var updateId = await _updateRepository.StartUpdateAsync( task.SourceSystem, task.SourceData, task.TableName, task.UpdateType, + parametersJson, cancellationToken); long recordCount = 0; @@ -133,6 +137,24 @@ public class TableSyncOperation : ITableSyncOperation } } + /// + /// Builds a JSON string of parameters for the DataUpdate record. + /// + private static string? BuildParametersJson(DataUpdateTask task) + { + var parametersDict = new Dictionary + { + ["OperationId"] = task.OperationId.ToString() + }; + + if (task.MinimumDt.HasValue) + { + parametersDict["MinimumDt"] = task.MinimumDt.Value.ToString("O"); + } + + return System.Text.Json.JsonSerializer.Serialize(parametersDict); + } + /// /// Core sync logic that uses the ETL pipeline. /// diff --git a/NEW/src/JdeScoping.Database/Scripts/002_CreateDataUpdateTable.sql b/NEW/src/JdeScoping.Database/Scripts/002_CreateDataUpdateTable.sql index e0b3bec..a7f92a6 100644 --- a/NEW/src/JdeScoping.Database/Scripts/002_CreateDataUpdateTable.sql +++ b/NEW/src/JdeScoping.Database/Scripts/002_CreateDataUpdateTable.sql @@ -15,6 +15,7 @@ BEGIN [UpdateType] SMALLINT NOT NULL, [WasSuccessful] BIT NOT NULL, [NumberRecords] BIGINT NOT NULL, + [Parameters] NVARCHAR(MAX) NULL, CONSTRAINT [PK_DataUpdate] PRIMARY KEY CLUSTERED([ID]) ); END diff --git a/NEW/src/JdeScoping.Database/Scripts/049_CreateProcessMisStagingDataProcedure.sql b/NEW/src/JdeScoping.Database/Scripts/049_CreateProcessMisStagingDataProcedure.sql new file mode 100644 index 0000000..a2a47d0 --- /dev/null +++ b/NEW/src/JdeScoping.Database/Scripts/049_CreateProcessMisStagingDataProcedure.sql @@ -0,0 +1,567 @@ +-- Migration: 049_CreateProcessMisStagingDataProcedure +-- Processes MIS staging data (mis_temp) into MisData_Curr and MisData_Hist tables +-- Handles version management with Current/BackLevel status tracking + +CREATE OR ALTER PROCEDURE [dbo].[usp_ProcessMisStagingData] + @SaveChanges BIT = 0 -- 0 = debug mode (SELECT only), 1 = save changes +AS +BEGIN + /* + ============================================= + Stored Procedure: usp_ProcessMisStagingData + ============================================= + + PURPOSE: + Processes MIS (Manufacturing Inspection Specification) data from the staging + table (mis_temp) into production tables (MisData_Curr and MisData_Hist). + Handles version management, ensuring only the latest version remains in _Curr + while older versions are archived to _Hist with proper status and dates. + + PARAMETERS: + @SaveChanges BIT (default 0) + 0 = Debug/preview mode - shows what would happen without making changes + 1 = Execute mode - applies all changes within a transaction + + TABLES INVOLVED: + - mis_temp : Staging table with incoming MIS data + - MisData_Curr : Current/active MIS specifications (Status = 'Current') + - MisData_Hist : Historical MIS specifications (Status = 'BackLevel') + + VERSION SCOPE: + Versions are scoped by: MisNumber + ItemNumber + BranchCode + Within each scope, RevID + ReleaseDate determine version ordering. + + PROCESSING FLOW: + + Step 0: Data Preparation + a) Clean staging data - remove 'IIS_' prefix from MisNumber values + b) Create performance indices on staging table + + Step 1-3: Version Analysis + 1) Build unified version list from MisData_Curr and mis_temp + 2) Identify the latest version for each scope + 3) Build version chain with calculated ObsoleteDates + (ObsoleteDate = ReleaseDate of the next version) + + Step 4-7: Record Classification + 4) Records to obsolete - Current records superseded by newer versions + 5) Intermediate versions - Staging records that are not the latest + 6) Records to merge - Existing current records with field changes + 7) Records to insert - New records for latest versions + + Step 8: Debug Output + - Displays summary counts and detailed record lists for review + + Step 9: Apply Changes (when @SaveChanges = 1) + a) UPDATE MisData_Curr: Set Status='BackLevel' and ObsoleteDate + for records being moved to history + b) INSERT INTO MisData_Hist: Copy BackLevel records from MisData_Curr + c) DELETE FROM MisData_Curr: Remove the BackLevel records + d) INSERT INTO MisData_Hist: Add intermediate versions with Status='BackLevel' + e) UPDATE MisData_Curr: Merge field changes for existing current records + f) INSERT INTO MisData_Curr: Add new records with Status='Current' + + STATUS VALUES: + - 'Current' : Active specification in MisData_Curr (ObsoleteDate = NULL) + - 'BackLevel' : Superseded specification in MisData_Hist (ObsoleteDate = set) + + ERROR HANDLING: + All changes in Step 9 are wrapped in a transaction. On error, changes are + rolled back and the error is re-thrown. + + USAGE: + -- Preview changes (debug mode) + EXEC usp_ProcessMisStagingData @SaveChanges = 0; + + -- Apply changes + EXEC usp_ProcessMisStagingData @SaveChanges = 1; + + ============================================= + */ + + SET NOCOUNT ON; + + -- ============================================= + -- Step 0a: Clean staging data - remove IIS_ prefix from MisNumber + -- ============================================= + + UPDATE mis_temp + SET MIS_IIS_Number = SUBSTRING(MIS_IIS_Number, 5, LEN(MIS_IIS_Number) - 4) + WHERE MIS_IIS_Number LIKE 'IIS_%'; + + PRINT 'Cleaned ' + CAST(@@ROWCOUNT AS VARCHAR(10)) + ' MisNumber values (removed IIS_ prefix)'; + + -- ============================================= + -- Step 0b: Recreate indices on staging table for performance + -- Naming convention: IX__ + -- ============================================= + + -- Drop existing indices if they exist + IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'IX_mis_temp_VersionLookup' AND object_id = OBJECT_ID('mis_temp')) + DROP INDEX IX_mis_temp_VersionLookup ON mis_temp; + + IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'IX_mis_temp_CharLookup' AND object_id = OBJECT_ID('mis_temp')) + DROP INDEX IX_mis_temp_CharLookup ON mis_temp; + + -- Also drop old index names if they exist (cleanup) + IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'IX_mis_temp_Version' AND object_id = OBJECT_ID('mis_temp')) + DROP INDEX IX_mis_temp_Version ON mis_temp; + + IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'IX_mis_temp_Full' AND object_id = OBJECT_ID('mis_temp')) + DROP INDEX IX_mis_temp_Full ON mis_temp; + + -- Create index for version-level joins (MisNumber + ItemNumber + BranchCode + RevID) + CREATE NONCLUSTERED INDEX IX_mis_temp_VersionLookup + ON mis_temp (MIS_IIS_Number, PartNumber, Site, Version) + INCLUDE (Release_Date); + + PRINT 'Created index IX_mis_temp_VersionLookup'; + + -- Create index for row-level joins (includes CharacterNumber) + CREATE NONCLUSTERED INDEX IX_mis_temp_CharLookup + ON mis_temp (MIS_IIS_Number, PartNumber, Site, Version, CharacterNumber) + INCLUDE (OperationNumber, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, Release_Date); + + PRINT 'Created index IX_mis_temp_CharLookup'; + + -- ============================================= + -- Step 1: Build unified version list from both MisData_Curr and mis_temp + -- Version scope: MisNumber + ItemNumber + BranchCode + -- ============================================= + + CREATE TABLE #AllVersions ( + MisNumber varchar(32), + ItemNumber varchar(32), + BranchCode varchar(32), + RevID varchar(32), + ReleaseDate datetime2, + Source varchar(10), + VersionRank int + ); + + INSERT INTO #AllVersions (MisNumber, ItemNumber, BranchCode, RevID, ReleaseDate, Source) + SELECT DISTINCT MisNumber, ItemNumber, BranchCode, RevID, ReleaseDate, 'CURR' + FROM MisData_Curr + WHERE Status = 'Current' AND ObsoleteDate IS NULL; + + INSERT INTO #AllVersions (MisNumber, ItemNumber, BranchCode, RevID, ReleaseDate, Source) + SELECT DISTINCT + LTRIM(RTRIM(MIS_IIS_Number)), + ISNULL(LTRIM(RTRIM(PartNumber)), '0'), + ISNULL(LTRIM(RTRIM(Site)), ''), + LTRIM(RTRIM(Version)), + Release_Date, + 'STAGING' + FROM mis_temp s + WHERE NOT EXISTS ( + SELECT 1 FROM #AllVersions a + WHERE a.MisNumber = LTRIM(RTRIM(s.MIS_IIS_Number)) + AND a.ItemNumber = ISNULL(LTRIM(RTRIM(s.PartNumber)), '0') + AND a.BranchCode = ISNULL(LTRIM(RTRIM(s.Site)), '') + AND a.RevID = LTRIM(RTRIM(s.Version)) + ); + + ;WITH RankedVersions AS ( + SELECT MisNumber, ItemNumber, BranchCode, RevID, ReleaseDate, Source, + ROW_NUMBER() OVER (PARTITION BY MisNumber, ItemNumber, BranchCode ORDER BY ReleaseDate, RevID) AS Rnk + FROM #AllVersions + ) + UPDATE a + SET VersionRank = r.Rnk + FROM #AllVersions a + INNER JOIN RankedVersions r ON + a.MisNumber = r.MisNumber + AND a.ItemNumber = r.ItemNumber + AND a.BranchCode = r.BranchCode + AND a.RevID = r.RevID; + + -- ============================================= + -- Step 2: Identify the latest version for each scope + -- ============================================= + + CREATE TABLE #LatestVersions ( + MisNumber varchar(32), + ItemNumber varchar(32), + BranchCode varchar(32), + RevID varchar(32), + ReleaseDate datetime2, + MaxRank int + ); + + INSERT INTO #LatestVersions + SELECT MisNumber, ItemNumber, BranchCode, RevID, ReleaseDate, VersionRank + FROM #AllVersions a + WHERE VersionRank = ( + SELECT MAX(VersionRank) FROM #AllVersions + WHERE MisNumber = a.MisNumber + AND ItemNumber = a.ItemNumber + AND BranchCode = a.BranchCode + ); + + -- ============================================= + -- Step 3: Build version chain with ObsoleteDates + -- ============================================= + + CREATE TABLE #VersionChain ( + MisNumber varchar(32), + ItemNumber varchar(32), + BranchCode varchar(32), + RevID varchar(32), + ReleaseDate datetime2, + ObsoleteDate datetime2, + Source varchar(10), + IsLatest bit + ); + + INSERT INTO #VersionChain + SELECT + a.MisNumber, a.ItemNumber, a.BranchCode, a.RevID, a.ReleaseDate, + (SELECT MIN(ReleaseDate) FROM #AllVersions + WHERE MisNumber = a.MisNumber + AND ItemNumber = a.ItemNumber + AND BranchCode = a.BranchCode + AND VersionRank = a.VersionRank + 1) AS ObsoleteDate, + a.Source, + CASE WHEN l.MisNumber IS NOT NULL THEN 1 ELSE 0 END AS IsLatest + FROM #AllVersions a + LEFT JOIN #LatestVersions l ON + a.MisNumber = l.MisNumber + AND a.ItemNumber = l.ItemNumber + AND a.BranchCode = l.BranchCode + AND a.RevID = l.RevID; + + CREATE NONCLUSTERED INDEX IX_VersionChain_Lookup ON #VersionChain (MisNumber, ItemNumber, BranchCode, RevID); + + -- ============================================= + -- Step 4: Identify records to move to history (from MisData_Curr) + -- These are current records whose version is superseded by a newer version + -- ============================================= + + CREATE TABLE #RecordsToObsolete ( + ItemNumber varchar(32), + BranchCode varchar(32), + SequenceNumber varchar(32), + MisNumber varchar(32), + RevID varchar(32), + CharNumber varchar(32), + TestDescription varchar(2000), + SamplingType varchar(32), + SamplingValue varchar(32), + ToolsGauges varchar(2000), + WorkInstructions varchar(2000), + Status varchar(32), + ReleaseDate datetime2, + ObsoleteDate datetime2 + ); + + INSERT INTO #RecordsToObsolete + SELECT + c.ItemNumber, c.BranchCode, c.SequenceNumber, c.MisNumber, c.RevID, c.CharNumber, + c.TestDescription, c.SamplingType, c.SamplingValue, c.ToolsGauges, c.WorkInstructions, + c.Status, c.ReleaseDate, + v.ObsoleteDate + FROM MisData_Curr c + INNER JOIN #VersionChain v ON + c.MisNumber = v.MisNumber + AND c.ItemNumber = v.ItemNumber + AND c.BranchCode = v.BranchCode + AND c.RevID = v.RevID + WHERE v.IsLatest = 0; + + -- ============================================= + -- Step 5: Identify intermediate versions from staging to insert into history + -- These are staging records for versions that are not the latest + -- ============================================= + + CREATE TABLE #IntermediateToHistory ( + ItemNumber varchar(32), + BranchCode varchar(32), + SequenceNumber varchar(32), + MisNumber varchar(32), + RevID varchar(32), + CharNumber varchar(32), + TestDescription varchar(2000), + SamplingType varchar(32), + SamplingValue varchar(32), + ToolsGauges varchar(2000), + WorkInstructions varchar(2000), + ReleaseDate datetime2, + ObsoleteDate datetime2 + ); + + INSERT INTO #IntermediateToHistory + SELECT + ISNULL(LTRIM(RTRIM(s.PartNumber)), '0'), + ISNULL(LTRIM(RTRIM(s.Site)), ''), + ISNULL(LTRIM(RTRIM(s.OperationNumber)), '0'), + LTRIM(RTRIM(s.MIS_IIS_Number)), + LTRIM(RTRIM(s.Version)), + LTRIM(RTRIM(s.CharacterNumber)), + LTRIM(RTRIM(s.TestDescription)), + LTRIM(RTRIM(s.SamplingType)), + LTRIM(RTRIM(s.SamplingValue)), + LTRIM(RTRIM(s.ToolsGauges)), + LTRIM(RTRIM(s.WorkInstructions)), + s.Release_Date, + v.ObsoleteDate + FROM mis_temp s + INNER JOIN #VersionChain v ON + LTRIM(RTRIM(s.MIS_IIS_Number)) = v.MisNumber + AND ISNULL(LTRIM(RTRIM(s.PartNumber)), '0') = v.ItemNumber + AND ISNULL(LTRIM(RTRIM(s.Site)), '') = v.BranchCode + AND LTRIM(RTRIM(s.Version)) = v.RevID + WHERE v.IsLatest = 0 AND v.Source = 'STAGING'; + + -- ============================================= + -- Step 6: Identify records to merge (update existing in MisData_Curr) + -- These are current records where field values have changed in staging + -- ============================================= + + CREATE TABLE #RecordsToMerge ( + MisNumber varchar(32), + ItemNumber varchar(32), + BranchCode varchar(32), + RevID varchar(32), + CharNumber varchar(32), + Curr_TestDescription varchar(2000), + Curr_SamplingType varchar(32), + Curr_SamplingValue varchar(32), + Curr_ToolsGauges varchar(2000), + Curr_WorkInstructions varchar(2000), + New_TestDescription varchar(2000), + New_SamplingType varchar(32), + New_SamplingValue varchar(32), + New_ToolsGauges varchar(2000), + New_WorkInstructions varchar(2000) + ); + + INSERT INTO #RecordsToMerge + SELECT + c.MisNumber, c.ItemNumber, c.BranchCode, c.RevID, c.CharNumber, + c.TestDescription, c.SamplingType, c.SamplingValue, c.ToolsGauges, c.WorkInstructions, + LTRIM(RTRIM(s.TestDescription)), LTRIM(RTRIM(s.SamplingType)), LTRIM(RTRIM(s.SamplingValue)), + LTRIM(RTRIM(s.ToolsGauges)), LTRIM(RTRIM(s.WorkInstructions)) + FROM MisData_Curr c + INNER JOIN mis_temp s ON + c.MisNumber = LTRIM(RTRIM(s.MIS_IIS_Number)) + AND c.ItemNumber = ISNULL(LTRIM(RTRIM(s.PartNumber)), '0') + AND c.BranchCode = ISNULL(LTRIM(RTRIM(s.Site)), '') + AND c.RevID = LTRIM(RTRIM(s.Version)) + AND c.CharNumber = LTRIM(RTRIM(s.CharacterNumber)) + INNER JOIN #VersionChain v ON + c.MisNumber = v.MisNumber + AND c.ItemNumber = v.ItemNumber + AND c.BranchCode = v.BranchCode + AND c.RevID = v.RevID + WHERE v.IsLatest = 1 + AND ( + ISNULL(c.TestDescription, '') <> ISNULL(LTRIM(RTRIM(s.TestDescription)), '') + OR ISNULL(c.SamplingType, '') <> ISNULL(LTRIM(RTRIM(s.SamplingType)), '') + OR ISNULL(c.SamplingValue, '') <> ISNULL(LTRIM(RTRIM(s.SamplingValue)), '') + OR ISNULL(c.ToolsGauges, '') <> ISNULL(LTRIM(RTRIM(s.ToolsGauges)), '') + OR ISNULL(c.WorkInstructions, '') <> ISNULL(LTRIM(RTRIM(s.WorkInstructions)), '') + ); + + -- ============================================= + -- Step 7: Identify new records to insert into MisData_Curr + -- These are staging records for the latest version that don't exist in current + -- ============================================= + + CREATE TABLE #RecordsToInsert ( + ItemNumber varchar(32), + BranchCode varchar(32), + SequenceNumber varchar(32), + MisNumber varchar(32), + RevID varchar(32), + CharNumber varchar(32), + TestDescription varchar(2000), + SamplingType varchar(32), + SamplingValue varchar(32), + ToolsGauges varchar(2000), + WorkInstructions varchar(2000), + ReleaseDate datetime2 + ); + + INSERT INTO #RecordsToInsert + SELECT + ISNULL(LTRIM(RTRIM(s.PartNumber)), '0'), + ISNULL(LTRIM(RTRIM(s.Site)), ''), + ISNULL(LTRIM(RTRIM(s.OperationNumber)), '0'), + LTRIM(RTRIM(s.MIS_IIS_Number)), + LTRIM(RTRIM(s.Version)), + LTRIM(RTRIM(s.CharacterNumber)), + LTRIM(RTRIM(s.TestDescription)), + LTRIM(RTRIM(s.SamplingType)), + LTRIM(RTRIM(s.SamplingValue)), + LTRIM(RTRIM(s.ToolsGauges)), + LTRIM(RTRIM(s.WorkInstructions)), + s.Release_Date + FROM mis_temp s + INNER JOIN #VersionChain v ON + LTRIM(RTRIM(s.MIS_IIS_Number)) = v.MisNumber + AND ISNULL(LTRIM(RTRIM(s.PartNumber)), '0') = v.ItemNumber + AND ISNULL(LTRIM(RTRIM(s.Site)), '') = v.BranchCode + AND LTRIM(RTRIM(s.Version)) = v.RevID + WHERE v.IsLatest = 1 + AND NOT EXISTS ( + SELECT 1 FROM MisData_Curr c + WHERE c.MisNumber = LTRIM(RTRIM(s.MIS_IIS_Number)) + AND c.ItemNumber = ISNULL(LTRIM(RTRIM(s.PartNumber)), '0') + AND c.BranchCode = ISNULL(LTRIM(RTRIM(s.Site)), '') + AND c.RevID = LTRIM(RTRIM(s.Version)) + AND c.CharNumber = LTRIM(RTRIM(s.CharacterNumber)) + ); + + -- ============================================= + -- Step 8: Output debug/preview information + -- ============================================= + + DECLARE @CountToObsolete INT = (SELECT COUNT(*) FROM #RecordsToObsolete); + DECLARE @CountIntermediate INT = (SELECT COUNT(*) FROM #IntermediateToHistory); + DECLARE @CountToMerge INT = (SELECT COUNT(*) FROM #RecordsToMerge); + DECLARE @CountToInsert INT = (SELECT COUNT(*) FROM #RecordsToInsert); + + PRINT '=== SUMMARY ==='; + PRINT 'Current records to move to history: ' + CAST(@CountToObsolete AS VARCHAR(10)); + PRINT 'Intermediate versions to history: ' + CAST(@CountIntermediate AS VARCHAR(10)); + PRINT 'Records to merge (update): ' + CAST(@CountToMerge AS VARCHAR(10)); + PRINT 'Records to insert (new): ' + CAST(@CountToInsert AS VARCHAR(10)); + + SELECT 'VERSION_CHAIN' AS Action, MisNumber, ItemNumber, BranchCode, RevID, + ReleaseDate, ObsoleteDate, Source, + CASE WHEN IsLatest = 1 THEN 'YES' ELSE 'NO' END AS IsLatest + FROM #VersionChain + WHERE EXISTS ( + SELECT 1 FROM #VersionChain vc + WHERE vc.MisNumber = #VersionChain.MisNumber + AND vc.ItemNumber = #VersionChain.ItemNumber + AND vc.BranchCode = #VersionChain.BranchCode + GROUP BY vc.MisNumber, vc.ItemNumber, vc.BranchCode + HAVING COUNT(*) > 1 + ) + ORDER BY MisNumber, ItemNumber, BranchCode, ReleaseDate; + + SELECT 'MOVE_TO_HIST' AS Action, MisNumber, ItemNumber, BranchCode, RevID, CharNumber, ReleaseDate, ObsoleteDate + FROM #RecordsToObsolete ORDER BY MisNumber, ItemNumber, BranchCode, RevID, CharNumber; + + SELECT 'INTERMEDIATE_TO_HIST' AS Action, MisNumber, ItemNumber, BranchCode, RevID, CharNumber, ReleaseDate, ObsoleteDate + FROM #IntermediateToHistory ORDER BY MisNumber, ItemNumber, BranchCode, RevID, CharNumber; + + SELECT 'MERGE_UPDATE' AS Action, MisNumber, ItemNumber, BranchCode, RevID, CharNumber, + Curr_TestDescription, New_TestDescription + FROM #RecordsToMerge ORDER BY MisNumber, ItemNumber, BranchCode, CharNumber; + + SELECT 'INSERT_NEW' AS Action, MisNumber, ItemNumber, BranchCode, RevID, CharNumber, ReleaseDate + FROM #RecordsToInsert ORDER BY MisNumber, ItemNumber, BranchCode, CharNumber; + + -- ============================================= + -- Step 9: If SaveChanges = 1, perform the operations + -- ============================================= + + IF @SaveChanges = 1 + BEGIN + BEGIN TRANSACTION; + + BEGIN TRY + -- 9a) Update records in MisData_Curr to BackLevel status before moving to history + UPDATE c SET + Status = 'BackLevel', + ObsoleteDate = o.ObsoleteDate + FROM MisData_Curr c + INNER JOIN #RecordsToObsolete o ON + c.MisNumber = o.MisNumber + AND c.ItemNumber = o.ItemNumber + AND c.BranchCode = o.BranchCode + AND c.RevID = o.RevID + AND c.CharNumber = o.CharNumber; + PRINT 'Updated ' + CAST(@@ROWCOUNT AS VARCHAR(10)) + ' records to BackLevel status in MisData_Curr'; + + -- 9b) Insert BackLevel records into history + INSERT INTO MisData_Hist ( + ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, + TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, + Status, ReleaseDate, ObsoleteDate + ) + SELECT + ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, + TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, + 'BackLevel', ReleaseDate, ObsoleteDate + FROM #RecordsToObsolete; + PRINT 'Moved ' + CAST(@@ROWCOUNT AS VARCHAR(10)) + ' records to MisData_Hist'; + + -- 9c) Delete BackLevel records from MisData_Curr + DELETE c FROM MisData_Curr c + INNER JOIN #RecordsToObsolete o ON + c.MisNumber = o.MisNumber + AND c.ItemNumber = o.ItemNumber + AND c.BranchCode = o.BranchCode + AND c.RevID = o.RevID + AND c.CharNumber = o.CharNumber; + PRINT 'Deleted ' + CAST(@@ROWCOUNT AS VARCHAR(10)) + ' BackLevel records from MisData_Curr'; + + -- 9d) Insert intermediate versions directly to history with BackLevel status + INSERT INTO MisData_Hist ( + ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, + TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, + Status, ReleaseDate, ObsoleteDate + ) + SELECT + ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, + TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, + 'BackLevel', ReleaseDate, ObsoleteDate + FROM #IntermediateToHistory; + PRINT 'Inserted ' + CAST(@@ROWCOUNT AS VARCHAR(10)) + ' intermediate versions into MisData_Hist'; + + -- 9e) Merge field changes for existing current records + UPDATE c SET + TestDescription = m.New_TestDescription, + SamplingType = m.New_SamplingType, + SamplingValue = m.New_SamplingValue, + ToolsGauges = m.New_ToolsGauges, + WorkInstructions = m.New_WorkInstructions + FROM MisData_Curr c + INNER JOIN #RecordsToMerge m ON + c.MisNumber = m.MisNumber + AND c.ItemNumber = m.ItemNumber + AND c.BranchCode = m.BranchCode + AND c.RevID = m.RevID + AND c.CharNumber = m.CharNumber; + PRINT 'Merged ' + CAST(@@ROWCOUNT AS VARCHAR(10)) + ' records in MisData_Curr'; + + -- 9f) Insert new records with Current status + INSERT INTO MisData_Curr ( + ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, + TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, + Status, ReleaseDate, ObsoleteDate + ) + SELECT + ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, + TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, + 'Current', ReleaseDate, NULL + FROM #RecordsToInsert; + PRINT 'Inserted ' + CAST(@@ROWCOUNT AS VARCHAR(10)) + ' new records into MisData_Curr'; + + COMMIT TRANSACTION; + PRINT 'Transaction committed successfully'; + END TRY + BEGIN CATCH + ROLLBACK TRANSACTION; + PRINT 'Error: ' + ERROR_MESSAGE(); + THROW; + END CATCH + END + ELSE + BEGIN + PRINT 'Debug mode: No changes saved. Set @SaveChanges = 1 to apply changes.'; + END + + -- Cleanup temp tables + DROP TABLE #RecordsToObsolete; + DROP TABLE #IntermediateToHistory; + DROP TABLE #VersionChain; + DROP TABLE #LatestVersions; + DROP TABLE #AllVersions; + DROP TABLE #RecordsToMerge; + DROP TABLE #RecordsToInsert; +END +GO diff --git a/NEW/src/JdeScoping.Host/JdeScoping.Host.csproj b/NEW/src/JdeScoping.Host/JdeScoping.Host.csproj index 4653504..120e6ce 100644 --- a/NEW/src/JdeScoping.Host/JdeScoping.Host.csproj +++ b/NEW/src/JdeScoping.Host/JdeScoping.Host.csproj @@ -16,6 +16,10 @@ + + + + net10.0 enable diff --git a/NEW/src/JdeScoping.Host/Pipelines/pipelines.json b/NEW/src/JdeScoping.Host/Pipelines/pipelines.json new file mode 100644 index 0000000..4fecb8f --- /dev/null +++ b/NEW/src/JdeScoping.Host/Pipelines/pipelines.json @@ -0,0 +1,392 @@ +{ + "settings": { + "timezone": "UTC" + }, + "scheduleDefaults": { + "mass": { "enabled": true, "intervalMinutes": 10080, "prePurge": true, "reIndex": true }, + "daily": { "enabled": true, "intervalMinutes": 1440, "prePurge": false, "reIndex": false }, + "hourly": { "enabled": true, "intervalMinutes": 60, "prePurge": false, "reIndex": false } + }, + "pipelines": { + "WorkOrder_Curr": { + "source": { + "connection": "jde", + "query": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo WHERE (wo.WAUPMJ > :dateUpdated OR (wo.WAUPMJ = :dateUpdated AND wo.WATDAY >= :timeUpdated))", + "massQuery": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "WorkOrder_Curr", + "matchColumns": ["WorkOrderNumber", "BranchCode"], + "excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "LastUpdateDt"] + } + }, + "Lot": { + "source": { + "connection": "jde", + "query": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL AND (lot.IOUPMJ > :dateUpdated OR (lot.IOUPMJ = :dateUpdated AND lot.IOTDAY >= :timeUpdated))", + "massQuery": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "Lot", + "matchColumns": ["LotNumber", "BranchCode"], + "excludeFromUpdate": ["LotNumber", "BranchCode", "LastUpdateDt"] + } + }, + "LotUsage_Curr": { + "source": { + "connection": "jde", + "query": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL AND (lu.ILTRDJ > :dateUpdated OR (lu.ILTRDJ = :dateUpdated AND lu.ILTDAY >= :timeUpdated))", + "massQuery": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "LotUsage_Curr", + "matchColumns": ["UniqueId"], + "excludeFromUpdate": ["UniqueId", "LastUpdateDt"] + } + }, + "Item": { + "source": { + "connection": "jde", + "query": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL AND (pn.IMUPMJ > :dateUpdated OR (pn.IMUPMJ = :dateUpdated AND pn.IMTDAY >= :timeUpdated))", + "massQuery": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "Item", + "matchColumns": ["ShortItemNumber"], + "excludeFromUpdate": ["ShortItemNumber", "LastUpdateDt"] + } + }, + "WorkCenter": { + "source": { + "connection": "jde", + "query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))", + "massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC'", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "WorkCenter", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + } + }, + "ProfitCenter": { + "source": { + "connection": "jde", + "query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))", + "massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3'", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "ProfitCenter", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + } + }, + "JdeUser": { + "source": { + "connection": "jde", + "query": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1", + "massQuery": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1", + "parameters": {} + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "JdeUser", + "matchColumns": ["AddressNumber"], + "excludeFromUpdate": ["AddressNumber", "LastUpdateDt"] + } + }, + "Branch": { + "source": { + "connection": "jde", + "query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))", + "massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP'", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "Branch", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + } + }, + "MisData_Curr": { + "source": { + "connection": "cms", + "query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current' AND Status.PDATE_RELEASED >= :lastUpdateDT", + "massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current'", + "parameters": { + "lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" } + } + }, + "schedules": { + "mass": { "intervalMinutes": 100800 }, + "daily": {}, + "hourly": { "enabled": false } + }, + "destination": { + "table": "MisData_Curr", + "matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"] + }, + "postScripts": [ + "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Curr.MisNumber = bl.MisNumber AND MisData_Curr.RevID = bl.RevID AND MisData_Curr.Status = 'Current' AND bl.Status = 'BackLevel';", + "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Curr.MisNumber = nl.MisNumber AND MisData_Curr.RevID < nl.RevID AND MisData_Curr.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;", + "MERGE INTO dbo.MisData_Hist AS target USING (SELECT * FROM dbo.MisData_Curr WHERE Status = 'BackLevel') AS source ON target.ItemNumber = source.ItemNumber AND target.BranchCode = source.BranchCode AND target.SequenceNumber = source.SequenceNumber AND target.MisNumber = source.MisNumber AND target.CharNumber = source.CharNumber WHEN MATCHED THEN UPDATE SET target.RevID = source.RevID, target.TestDescription = source.TestDescription, target.SamplingType = source.SamplingType, target.SamplingValue = source.SamplingValue, target.ToolsGauges = source.ToolsGauges, target.WorkInstructions = source.WorkInstructions, target.Status = source.Status, target.ReleaseDate = source.ReleaseDate, target.ObsoleteDate = source.ObsoleteDate WHEN NOT MATCHED THEN INSERT (ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, Status, ReleaseDate, ObsoleteDate) VALUES (source.ItemNumber, source.BranchCode, source.SequenceNumber, source.MisNumber, source.RevID, source.CharNumber, source.TestDescription, source.SamplingType, source.SamplingValue, source.ToolsGauges, source.WorkInstructions, source.Status, source.ReleaseDate, source.ObsoleteDate);", + "DELETE FROM dbo.MisData_Curr WHERE Status = 'BackLevel';", + "ALTER INDEX [PK_MisData_Curr] ON [dbo].[MisData_Curr] REBUILD;" + ] + }, + "MisData_Hist": { + "source": { + "connection": "cms", + "query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel' AND Status.PDATE_RELEASED >= :lastUpdateDT", + "massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel'", + "parameters": { + "lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" } + } + }, + "schedules": { + "mass": { "intervalMinutes": 100800 }, + "daily": { "enabled": false }, + "hourly": { "enabled": false } + }, + "destination": { + "table": "MisData_Hist", + "matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"] + }, + "postScripts": [ + "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Hist.MisNumber = bl.MisNumber AND MisData_Hist.RevID = bl.RevID AND MisData_Hist.Status = 'Current' AND bl.Status = 'BackLevel';", + "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Hist.MisNumber = nl.MisNumber AND MisData_Hist.RevID < nl.RevID AND MisData_Hist.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;", + "ALTER INDEX [PK_MisData_Hist] ON [dbo].[MisData_Hist] REBUILD;" + ] + }, + "WorkOrderTime_Curr": { + "source": { + "connection": "jde", + "query": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot WHERE (wot.DATEUPDATED_WTUPMJ > :dateUpdated OR (wot.DATEUPDATED_WTUPMJ = :dateUpdated AND wot.TIMEOFDAY_WTTDAY >= :timeUpdated))", + "massQuery": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "WorkOrderTime_Curr", + "matchColumns": ["UniqueID"], + "excludeFromUpdate": ["UniqueID", "LastUpdateDt"] + } + }, + "WorkOrderComponent_Curr": { + "source": { + "connection": "jde", + "query": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL AND (woc.DATEUPDATED_WMUPMJ > :dateUpdated OR (woc.DATEUPDATED_WMUPMJ = :dateUpdated AND woc.TIMEOFDAY_WMTDAY >= :timeUpdated))", + "massQuery": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "WorkOrderComponent_Curr", + "matchColumns": ["UniqueID"], + "excludeFromUpdate": ["UniqueID", "LastUpdateDt"] + } + }, + "WorkOrderStep_Curr": { + "source": { + "connection": "jde", + "query": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY) WHERE (wos.DATEUPDATED_WLUPMJ > :dateUpdated OR (wos.DATEUPDATED_WLUPMJ = :dateUpdated AND wos.TIMEOFDAY_WLTDAY >= :timeUpdated))", + "massQuery": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY)", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "WorkOrderStep_Curr", + "matchColumns": ["WorkOrderNumber", "BranchCode", "StepNumber"], + "excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "StepNumber", "LastUpdateDt"] + } + }, + "WorkOrderRouting": { + "source": { + "connection": "jde", + "query": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410' AND (woz.DATEUPDATED_SZUPMJ > :dateUpdated OR (woz.DATEUPDATED_SZUPMJ = :dateUpdated AND woz.TIMEOFDAY_SZTDAY >= :timeUpdated))", + "massQuery": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410'", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "WorkOrderRouting", + "matchColumns": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber"], + "excludeFromUpdate": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber", "LastUpdateDt"] + } + }, + "StatusCode": { + "source": { + "connection": "giw", + "query": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL AND (sc.DATEUPDATED_DRUPMJ > :dateUpdated OR (sc.DATEUPDATED_DRUPMJ = :dateUpdated AND sc.TIMELASTUPDATED_DRUPMT >= :timeUpdated))", + "massQuery": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "StatusCode", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + } + }, + "OrgHierarchy": { + "source": { + "connection": "jde", + "query": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL AND (oh.DATEUPDATED_IWUPMJ > :dateUpdated OR (oh.DATEUPDATED_IWUPMJ = :dateUpdated AND oh.TIMEOFDAY_IWTDAY >= :timeUpdated))", + "massQuery": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "OrgHierarchy", + "matchColumns": ["WorkCenterCode", "BranchCode"], + "excludeFromUpdate": ["WorkCenterCode", "BranchCode", "LastUpdateDt"] + } + }, + "RouteMaster": { + "source": { + "connection": "jde", + "query": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL AND (route_master.DATEUPDATED_IRUPMJ > :dateUpdated OR (route_master.DATEUPDATED_IRUPMJ = :dateUpdated AND route_master.TIMEOFDAY_IRTDAY >= :timeUpdated))", + "massQuery": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL", + "parameters": { + "dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" }, + "timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" } + } + }, + "schedules": { + "mass": {}, + "daily": {}, + "hourly": {} + }, + "destination": { + "table": "RouteMaster", + "matchColumns": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber"], + "excludeFromUpdate": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber", "LastUpdateDt"] + } + }, + "FunctionCode": { + "source": { + "connection": "jde", + "query": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code", + "massQuery": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code", + "parameters": {} + }, + "schedules": { + "mass": { "prePurge": true, "reIndex": true }, + "daily": { "prePurge": true, "reIndex": true }, + "hourly": { "prePurge": true, "reIndex": true } + }, + "destination": { + "table": "FunctionCode", + "matchColumns": ["Code"], + "excludeFromUpdate": ["Code", "LastUpdateDt"] + } + } + } +} diff --git a/NEW/src/JdeScoping.Host/Program.cs b/NEW/src/JdeScoping.Host/Program.cs index 419df25..c9f6f3c 100644 --- a/NEW/src/JdeScoping.Host/Program.cs +++ b/NEW/src/JdeScoping.Host/Program.cs @@ -3,7 +3,7 @@ using JdeScoping.DataAccess.Options; using JdeScoping.DataSync.Options; using JdeScoping.ExcelIO.Options; using JdeScoping.Database; -using JdeScoping.Infrastructure.Security; +using JdeScoping.Host.Startup; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -45,11 +45,17 @@ builder.Services var app = builder.Build(); -// Migrate existing secrets to SecureStore (skip in Testing environment) +// Configuration validation (skip in Testing environment) if (!app.Environment.IsEnvironment("Testing")) { - var migrator = app.Services.GetRequiredService(); - migrator.MigrateIfNeeded(); + var configLogger = app.Services.GetRequiredService() + .CreateLogger("JdeScoping.Host.ConfigurationValidation"); + + if (!ConfigurationValidationRunner.ValidateConfiguration(app.Services, configLogger)) + { + configLogger.LogCritical("Application startup aborted due to configuration validation failures"); + return 1; + } } // Startup validation - verify critical services are registered diff --git a/NEW/src/JdeScoping.Host/Startup/ConfigurationValidationRunner.cs b/NEW/src/JdeScoping.Host/Startup/ConfigurationValidationRunner.cs new file mode 100644 index 0000000..8085e78 --- /dev/null +++ b/NEW/src/JdeScoping.Host/Startup/ConfigurationValidationRunner.cs @@ -0,0 +1,81 @@ +using JdeScoping.Core.Validation; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace JdeScoping.Host.Startup; + +/// +/// Runs all registered configuration validators on application startup. +/// +public static class ConfigurationValidationRunner +{ + /// + /// Validates configuration using all registered IConfigurationValidator implementations. + /// + /// The service provider. + /// Logger for validation output. + /// True if all validators pass, false if any errors occurred. + public static bool ValidateConfiguration(IServiceProvider services, ILogger logger) + { + var validators = services.GetServices() + .OrderBy(v => v.Order) + .ToList(); + + if (validators.Count == 0) + { + logger.LogDebug("No configuration validators registered"); + return true; + } + + logger.LogInformation("Running {Count} configuration validator(s)", validators.Count); + + var results = new List(); + foreach (var validator in validators) + { + try + { + logger.LogDebug("Running validator: {Name} (Order={Order})", validator.Name, validator.Order); + var result = validator.Validate(); + results.Add(result); + + // Log warnings immediately + foreach (var warning in result.Warnings) + { + logger.LogWarning("[{ValidatorName}] {Warning}", validator.Name, warning); + } + } + catch (Exception ex) + { + logger.LogError(ex, "Validator {Name} threw an exception", validator.Name); + var errorResult = new ConfigurationValidationResult(validator.Name); + errorResult.AddError($"Validator threw an exception: {ex.Message}"); + results.Add(errorResult); + } + } + + var failedResults = results.Where(r => !r.IsValid).ToList(); + if (failedResults.Count == 0) + { + logger.LogInformation("Configuration validation passed"); + return true; + } + + // Log all errors + var totalErrors = failedResults.Sum(r => r.Errors.Count); + logger.LogCritical( + "Configuration validation FAILED with {ErrorCount} error(s) from {ValidatorCount} validator(s)", + totalErrors, + failedResults.Count); + + foreach (var result in failedResults) + { + logger.LogCritical("=== {ValidatorName} Validator Errors ===", result.ValidatorName); + foreach (var error in result.Errors) + { + logger.LogCritical(" - {Error}", error); + } + } + + return false; + } +} diff --git a/NEW/src/JdeScoping.Host/appsettings.json b/NEW/src/JdeScoping.Host/appsettings.json index 50cbfc0..8e442bf 100644 --- a/NEW/src/JdeScoping.Host/appsettings.json +++ b/NEW/src/JdeScoping.Host/appsettings.json @@ -92,13 +92,22 @@ "HourlyConfig": { "Enabled": false } }, { - "TableName": "MisData", + "TableName": "MisData_Curr", "SourceSystem": "CMS", - "SourceData": "MISDATA", + "SourceData": "MISDATA_CURR", "IsEnabled": true, "MassConfig": { "Enabled": true, "IntervalMinutes": 10080 }, "DailyConfig": { "Enabled": true, "IntervalMinutes": 1440 }, "HourlyConfig": { "Enabled": false } + }, + { + "TableName": "MisData_Hist", + "SourceSystem": "CMS", + "SourceData": "MISDATA_HIST", + "IsEnabled": true, + "MassConfig": { "Enabled": true, "IntervalMinutes": 10080 }, + "DailyConfig": { "Enabled": false }, + "HourlyConfig": { "Enabled": false } } ] }, @@ -133,7 +142,11 @@ "KeyFilePath": "data/secrets.key", "MasterKeyEnvVar": "SCOPINGTOOL_MASTER_KEY", "AutoCreateStore": true, - "MigrateExistingSecrets": true + "RequiredKeys": [ + "RsaPrivateKey", + "ExcelExport:CriteriaSheetPassword", + "ExcelExport:DataSheetPassword" + ] }, "WorkProcessor": { "Enabled": true, diff --git a/NEW/src/JdeScoping.Infrastructure/DependencyInjection.cs b/NEW/src/JdeScoping.Infrastructure/DependencyInjection.cs index 311834b..ec1246f 100644 --- a/NEW/src/JdeScoping.Infrastructure/DependencyInjection.cs +++ b/NEW/src/JdeScoping.Infrastructure/DependencyInjection.cs @@ -1,8 +1,10 @@ using JdeScoping.Core.Interfaces; using JdeScoping.Core.Options; +using JdeScoping.Core.Validation; using JdeScoping.Infrastructure.Auth; using JdeScoping.Infrastructure.Options; using JdeScoping.Infrastructure.Security; +using JdeScoping.Infrastructure.Validation; using Microsoft.Extensions.Configuration; namespace Microsoft.Extensions.DependencyInjection; @@ -52,8 +54,9 @@ public static class InfrastructureDependencyInjection services.AddSingleton(); - // Register secrets migrator for one-time migration of existing secrets - services.AddSingleton(); + // Register configuration validators + services.AddSingleton(); + services.AddSingleton(); return services; } diff --git a/NEW/src/JdeScoping.Infrastructure/Security/SecretsMigrator.cs b/NEW/src/JdeScoping.Infrastructure/Security/SecretsMigrator.cs deleted file mode 100644 index 5807241..0000000 --- a/NEW/src/JdeScoping.Infrastructure/Security/SecretsMigrator.cs +++ /dev/null @@ -1,161 +0,0 @@ -using System.Security.Cryptography; -using JdeScoping.Core.Interfaces; -using JdeScoping.Core.Options; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace JdeScoping.Infrastructure.Security; - -/// -/// Migrates existing secrets to SecureStore on first run. -/// -public class SecretsMigrator -{ - private readonly ISecureStoreService _secureStore; - private readonly IConfiguration _configuration; - private readonly SecureStoreOptions _options; - private readonly ILogger _logger; - - // Well-known secret keys - public const string RsaPrivateKeyName = "RsaPrivateKey"; - public const string ExcelCriteriaPasswordKey = "ExcelExport:CriteriaSheetPassword"; - public const string ExcelDataPasswordKey = "ExcelExport:DataSheetPassword"; - - /// - /// Initializes a new instance of the class. - /// - /// Service for secure secret storage. - /// Application configuration containing existing secrets. - /// Options controlling secret migration behavior. - /// Logger for recording migration operations. - public SecretsMigrator( - ISecureStoreService secureStore, - IConfiguration configuration, - IOptions options, - ILogger logger) - { - _secureStore = secureStore; - _configuration = configuration; - _options = options.Value; - _logger = logger; - } - - /// - /// Migrates existing secrets if migration is enabled and secrets haven't been migrated yet. - /// - public void MigrateIfNeeded() - { - if (!_options.MigrateExistingSecrets) - { - _logger.LogDebug("Secret migration is disabled"); - return; - } - - var migrated = false; - - migrated |= MigrateRsaKey(); - migrated |= MigrateExcelPasswords(); - - if (migrated) - { - // Save with metadata to persist keys list - if (_secureStore is SecureStoreService sss) - { - sss.SaveWithMetadata(); - } - else - { - _secureStore.Save(); - } - - _logger.LogInformation("Secret migration completed"); - } - } - - private bool MigrateRsaKey() - { - // Skip if already in SecureStore - if (_secureStore.Contains(RsaPrivateKeyName)) - { - _logger.LogDebug("RSA key already in SecureStore, skipping migration"); - return false; - } - - // Look for existing key file - var rsaKeyOptions = _configuration.GetSection("RsaKey").Get() ?? new RsaKeyOptions(); - var keyFilePath = Path.IsPathRooted(rsaKeyOptions.KeyFilePath) - ? rsaKeyOptions.KeyFilePath - : Path.Combine(AppContext.BaseDirectory, rsaKeyOptions.KeyFilePath); - - if (!File.Exists(keyFilePath)) - { - _logger.LogDebug("No existing RSA key file found at {KeyFilePath}", keyFilePath); - return false; - } - - try - { - // Read the binary RSA key file - var keyBytes = File.ReadAllBytes(keyFilePath); - - // Import into RSA and export as PEM - using var rsa = RSA.Create(); - rsa.ImportRSAPrivateKey(keyBytes, out _); - var pemKey = rsa.ExportRSAPrivateKeyPem(); - - // Store in SecureStore - _secureStore.Set(RsaPrivateKeyName, pemKey); - _logger.LogInformation("Migrated RSA key from {KeyFilePath} to SecureStore", keyFilePath); - - // Optionally delete the old file - try - { - File.Delete(keyFilePath); - _logger.LogInformation("Deleted old RSA key file at {KeyFilePath}", keyFilePath); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Could not delete old RSA key file at {KeyFilePath}", keyFilePath); - } - - return true; - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to migrate RSA key from {KeyFilePath}", keyFilePath); - return false; - } - } - - private bool MigrateExcelPasswords() - { - var migrated = false; - - // Migrate criteria sheet password if configured and not already in SecureStore - if (!_secureStore.Contains(ExcelCriteriaPasswordKey)) - { - var password = _configuration["ExcelExport:CriteriaSheetPassword"]; - if (!string.IsNullOrEmpty(password) && password != string.Empty) - { - _secureStore.Set(ExcelCriteriaPasswordKey, password); - _logger.LogInformation("Migrated Excel criteria sheet password to SecureStore"); - migrated = true; - } - } - - // Migrate data sheet password if configured and not already in SecureStore - if (!_secureStore.Contains(ExcelDataPasswordKey)) - { - var password = _configuration["ExcelExport:DataSheetPassword"]; - if (!string.IsNullOrEmpty(password) && password != string.Empty) - { - _secureStore.Set(ExcelDataPasswordKey, password); - _logger.LogInformation("Migrated Excel data sheet password to SecureStore"); - migrated = true; - } - } - - return migrated; - } -} diff --git a/NEW/src/JdeScoping.Infrastructure/Validation/LdapOptionsValidator.cs b/NEW/src/JdeScoping.Infrastructure/Validation/LdapOptionsValidator.cs new file mode 100644 index 0000000..ad11cf5 --- /dev/null +++ b/NEW/src/JdeScoping.Infrastructure/Validation/LdapOptionsValidator.cs @@ -0,0 +1,58 @@ +using JdeScoping.Core.Validation; +using JdeScoping.Infrastructure.Options; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace JdeScoping.Infrastructure.Validation; + +/// +/// Validates LDAP configuration options. +/// +public class LdapOptionsValidator : IConfigurationValidator +{ + private readonly LdapOptions _options; + private readonly ILogger _logger; + + /// + public int Order => 200; + + /// + public string Name => "LdapOptions"; + + public LdapOptionsValidator( + IOptions options, + ILogger logger) + { + _options = options.Value; + _logger = logger; + } + + /// + public ConfigurationValidationResult Validate() + { + var result = new ConfigurationValidationResult(Name); + + if (_options.UseFakeAuth) + { + _logger.LogDebug("UseFakeAuth is enabled, skipping LDAP validation"); + return result; + } + + if (_options.ServerUrls.Length == 0 || _options.ServerUrls.All(string.IsNullOrWhiteSpace)) + { + result.AddError("ServerUrls must contain at least one LDAP server URL when UseFakeAuth is false"); + } + + if (string.IsNullOrWhiteSpace(_options.GroupDn)) + { + result.AddError("GroupDn is required when UseFakeAuth is false"); + } + + if (string.IsNullOrWhiteSpace(_options.SearchBase)) + { + result.AddError("SearchBase is required when UseFakeAuth is false"); + } + + return result; + } +} diff --git a/NEW/src/JdeScoping.Infrastructure/Validation/SecureStoreValidator.cs b/NEW/src/JdeScoping.Infrastructure/Validation/SecureStoreValidator.cs new file mode 100644 index 0000000..1e0fcb8 --- /dev/null +++ b/NEW/src/JdeScoping.Infrastructure/Validation/SecureStoreValidator.cs @@ -0,0 +1,82 @@ +using JdeScoping.Core.Interfaces; +using JdeScoping.Core.Options; +using JdeScoping.Core.Validation; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace JdeScoping.Infrastructure.Validation; + +/// +/// Validates that required secrets exist in the SecureStore. +/// +public class SecureStoreValidator : IConfigurationValidator +{ + private readonly ISecureStoreService _secureStore; + private readonly SecureStoreOptions _options; + private readonly ILogger _logger; + + /// + public int Order => 100; + + /// + public string Name => "SecureStore"; + + public SecureStoreValidator( + ISecureStoreService secureStore, + IOptions options, + ILogger logger) + { + _secureStore = secureStore; + _options = options.Value; + _logger = logger; + } + + /// + public ConfigurationValidationResult Validate() + { + var result = new ConfigurationValidationResult(Name); + + if (_options.RequiredKeys.Count == 0) + { + _logger.LogDebug("No required keys configured, skipping SecureStore validation"); + return result; + } + + _logger.LogDebug("Validating {Count} required secrets", _options.RequiredKeys.Count); + + foreach (var key in _options.RequiredKeys) + { + if (string.IsNullOrWhiteSpace(key)) + { + result.AddWarning("Empty key in RequiredKeys configuration"); + continue; + } + + if (!_secureStore.Contains(key)) + { + result.AddError($"Required secret '{key}' not found in SecureStore"); + continue; + } + + // Verify the secret can actually be retrieved + try + { + var value = _secureStore.Get(key); + if (string.IsNullOrEmpty(value)) + { + result.AddError($"Required secret '{key}' exists but has an empty value"); + } + else + { + _logger.LogDebug("Validated required secret: {Key}", key); + } + } + catch (Exception ex) + { + result.AddError($"Required secret '{key}' could not be retrieved: {ex.Message}"); + } + } + + return result; + } +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Models/PipelineModel.cs b/NEW/src/Utils/JdeScoping.ConfigManager/Models/PipelineModel.cs index 325f6e5..05eff08 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Models/PipelineModel.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Models/PipelineModel.cs @@ -59,11 +59,21 @@ public class PipelineModel /// public PipelineSchedules Schedules { get; set; } = new(); + /// + /// Gets or sets optional data transformers applied between source and destination. + /// + public List? Transformers { get; set; } + /// /// Gets or sets the destination configuration for data loading. /// public PipelineDestination Destination { get; set; } = new(); + /// + /// Gets or sets optional scripts to execute before pipeline starts. + /// + public string[]? PreScripts { get; set; } + /// /// Gets or sets optional scripts to execute after pipeline completion. /// @@ -74,23 +84,33 @@ public class PipelineSource { /// /// Gets or sets the source database connection name. + /// Used for database sources. /// public string Connection { get; set; } = string.Empty; /// /// Gets or sets the query to extract data from the source. + /// Used for database sources. /// public string Query { get; set; } = string.Empty; /// /// Gets or sets the optional mass query for full data extraction. + /// Used for database sources. /// public string? MassQuery { get; set; } /// /// Gets or sets the query parameters and their definitions. + /// Used for database sources. /// public Dictionary Parameters { get; set; } = new(); + + /// + /// Gets or sets the file name for file-based sources. + /// Used for Protobuf+Zstd files. + /// + public string? FileName { get; set; } } public class ParameterDefinition @@ -131,6 +151,12 @@ public class PipelineSchedules public class PipelineDestination { + /// + /// Gets or sets the destination type (BulkImport or BulkMerge). + /// BulkImport truncates and loads; BulkMerge matches and updates. + /// + public string Type { get; set; } = "BulkMerge"; + /// /// Gets or sets the destination table name. /// @@ -138,11 +164,52 @@ public class PipelineDestination /// /// Gets or sets the columns used to match existing records for updates. + /// Only used for BulkMerge destination type. /// public string[] MatchColumns { get; set; } = []; /// /// Gets or sets the columns to exclude from update operations. + /// Only used for BulkMerge destination type. /// public string[] ExcludeFromUpdate { get; set; } = []; } + +/// +/// Represents a data transformer applied between source and destination. +/// +public class TransformerModel +{ + /// + /// Gets or sets the transformer type. + /// Supported types: ColumnDrop, ColumnRename, JdeDate. + /// + public string Type { get; set; } = string.Empty; + + /// + /// Gets or sets the columns affected by this transformer. + /// Used by ColumnDrop (columns to remove) and JdeDate (date/time columns). + /// + public List? Columns { get; set; } + + /// + /// Gets or sets the column mappings for rename operations. + /// Used by ColumnRename: OldName → NewName. + /// + public Dictionary? Mappings { get; set; } + + /// + /// Gets or sets the date column name for JdeDate transformer. + /// + public string? DateColumn { get; set; } + + /// + /// Gets or sets the time column name for JdeDate transformer. + /// + public string? TimeColumn { get; set; } + + /// + /// Gets or sets the output column name for JdeDate transformer. + /// + public string? OutputColumn { get; set; } +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Services/AutoDiscoveryService.cs b/NEW/src/Utils/JdeScoping.ConfigManager/Services/AutoDiscoveryService.cs index a95520d..f1ec81f 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Services/AutoDiscoveryService.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Services/AutoDiscoveryService.cs @@ -59,7 +59,26 @@ public class AutoDiscoveryService : IAutoDiscoveryService return Task.FromResult(hostDir); } - // 4. Check user config directory + // 4. Check project structure paths (for development) + // When running from bin/Debug/net10.0, go up to find src/JdeScoping.Host + var projectHostPaths = new[] + { + // From bin/Debug/net10.0 -> src/JdeScoping.Host + _fileSystem.Combine(exeDir, "..", "..", "..", "..", "..", "JdeScoping.Host"), + // Absolute fallback for development + "/Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.Host" + }; + + foreach (var projectPath in projectHostPaths) + { + if (IsValidConfigFolder(projectPath)) + { + _logger?.LogInformation("Found config folder in project directory: {Path}", projectPath); + return Task.FromResult(Path.GetFullPath(projectPath)); + } + } + + // 5. Check user config directory var userConfigDir = GetUserConfigDirectory(); if (userConfigDir != null && IsValidConfigFolder(userConfigDir)) { diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Services/AvaloniaDialogService.cs b/NEW/src/Utils/JdeScoping.ConfigManager/Services/AvaloniaDialogService.cs index cbd42e5..3df6570 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Services/AvaloniaDialogService.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Services/AvaloniaDialogService.cs @@ -1,6 +1,7 @@ using System.Text; using Avalonia.Controls; using Avalonia.Platform.Storage; +using JdeScoping.ConfigManager.Views.Dialogs; using MsBox.Avalonia; using MsBox.Avalonia.Enums; @@ -146,4 +147,22 @@ public class AvaloniaDialogService : IDialogService await box.ShowAsync(); } } + + /// + public async Task ShowInputDialogAsync(string title, string prompt, string? defaultValue = null) + { + var window = _getMainWindow(); + if (window == null) + return null; + + var dialog = new InputDialog(title, prompt, defaultValue); + var result = await dialog.ShowDialog(window); + + if (result == true) + { + return dialog.InputText; + } + + return null; + } } diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Services/IDialogService.cs b/NEW/src/Utils/JdeScoping.ConfigManager/Services/IDialogService.cs index 16786f5..bce7db0 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Services/IDialogService.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Services/IDialogService.cs @@ -42,4 +42,13 @@ public interface IDialogService /// Validation result for appsettings.json. /// Validation result for pipelines.json. Task ShowValidationResultsAsync(ValidationResult appSettingsResult, ValidationResult pipelinesResult); + + /// + /// Shows an input dialog to collect text from the user. + /// + /// The dialog title. + /// The prompt message to display. + /// Optional default value for the input field. + /// The text entered by the user, or null if cancelled. + Task ShowInputDialogAsync(string title, string prompt, string? defaultValue = null); } diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/Forms/PipelineEditorViewModel.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/Forms/PipelineEditorViewModel.cs new file mode 100644 index 0000000..eafc021 --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/Forms/PipelineEditorViewModel.cs @@ -0,0 +1,443 @@ +using JdeScoping.ConfigManager.Models; +using JdeScoping.ConfigManager.ViewModels.PipelineSteps; +using System.Collections.ObjectModel; +using System.Windows.Input; + +namespace JdeScoping.ConfigManager.ViewModels.Forms; + +/// +/// ViewModel for the visual pipeline editor with flow diagram. +/// +public class PipelineEditorViewModel : ViewModelBase +{ + private readonly PipelineModel _model; + private readonly Action _onChanged; + private PipelineStepViewModelBase? _selectedStep; + private object? _selectedStepEditor; + + public PipelineEditorViewModel(string name, PipelineModel model, IReadOnlyList availableConnections, Action onChanged) + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + _model = model ?? throw new ArgumentNullException(nameof(model)); + AvailableConnections = availableConnections ?? []; + _onChanged = onChanged ?? throw new ArgumentNullException(nameof(onChanged)); + + // Initialize collections + PreScripts = []; + Transformers = []; + PostScripts = []; + + // Initialize schedule view models + _model.Schedules.Mass ??= new ScheduleModel(); + _model.Schedules.Daily ??= new ScheduleModel(); + _model.Schedules.Hourly ??= new ScheduleModel(); + + MassSchedule = new ScheduleFormViewModel(_model.Schedules.Mass, _onChanged); + DailySchedule = new ScheduleFormViewModel(_model.Schedules.Daily, _onChanged); + HourlySchedule = new ScheduleFormViewModel(_model.Schedules.Hourly, _onChanged); + + // Build the pipeline steps from the model + BuildPipelineSteps(); + + // Initialize commands + AddPreScriptCommand = new RelayCommand(AddPreScript); + AddTransformerCommand = new RelayCommand(AddTransformer); + AddPostScriptCommand = new RelayCommand(AddPostScript); + RemoveStepCommand = new RelayCommand(RemoveStep); + MoveStepUpCommand = new RelayCommand(MoveStepUp, CanMoveStepUp); + MoveStepDownCommand = new RelayCommand(MoveStepDown, CanMoveStepDown); + } + + /// + /// Gets the pipeline name. + /// + public string Name { get; } + + /// + /// Gets the available connection names from configuration. + /// + public IReadOnlyList AvailableConnections { get; } + + /// + /// Gets the pre-script steps. + /// + public ObservableCollection PreScripts { get; } + + /// + /// Gets the source step. + /// + public SourceStepViewModel Source { get; private set; } = null!; + + /// + /// Gets the transformer steps. + /// + public ObservableCollection Transformers { get; } + + /// + /// Gets the destination step. + /// + public DestinationStepViewModel Destination { get; private set; } = null!; + + /// + /// Gets the post-script steps. + /// + public ObservableCollection PostScripts { get; } + + /// + /// Gets all pipeline steps in flow order for display. + /// + public IEnumerable AllSteps + { + get + { + foreach (var step in PreScripts) + yield return step; + yield return Source; + foreach (var step in Transformers) + yield return step; + yield return Destination; + foreach (var step in PostScripts) + yield return step; + } + } + + /// + /// Gets or sets the currently selected pipeline step. + /// + public PipelineStepViewModelBase? SelectedStep + { + get => _selectedStep; + set + { + if (_selectedStep != value) + { + // Deselect previous step + if (_selectedStep != null) + _selectedStep.IsSelected = false; + + _selectedStep = value; + + // Select new step + if (_selectedStep != null) + _selectedStep.IsSelected = true; + + OnPropertyChanged(); + UpdateSelectedStepEditor(); + } + } + } + + /// + /// Gets the editor view model for the currently selected step. + /// + public object? SelectedStepEditor + { + get => _selectedStepEditor; + private set => SetProperty(ref _selectedStepEditor, value); + } + + /// + /// Gets the mass schedule view model. + /// + public ScheduleFormViewModel MassSchedule { get; } + + /// + /// Gets the daily schedule view model. + /// + public ScheduleFormViewModel DailySchedule { get; } + + /// + /// Gets the hourly schedule view model. + /// + public ScheduleFormViewModel HourlySchedule { get; } + + // Commands + public ICommand AddPreScriptCommand { get; } + public ICommand AddTransformerCommand { get; } + public ICommand AddPostScriptCommand { get; } + public ICommand RemoveStepCommand { get; } + public ICommand MoveStepUpCommand { get; } + public ICommand MoveStepDownCommand { get; } + + /// + /// Gets the list of available transformer types for the add dialog. + /// + public IReadOnlyList AvailableTransformerTypes => TransformerFactory.AvailableTypes; + + /// + /// Property to track selected transformer type for adding. + /// + private string? _selectedTransformerType; + public string? SelectedTransformerType + { + get => _selectedTransformerType; + set => SetProperty(ref _selectedTransformerType, value); + } + + private void BuildPipelineSteps() + { + // Pre-scripts + PreScripts.Clear(); + if (_model.PreScripts != null) + { + foreach (var script in _model.PreScripts) + { + PreScripts.Add(new PreScriptStepViewModel(script, () => + { + SyncPreScriptsToModel(); + _onChanged(); + })); + } + } + + // Source + Source = new SourceStepViewModel(_model.Source, () => + { + _onChanged(); + }); + + // Transformers + Transformers.Clear(); + if (_model.Transformers != null) + { + foreach (var transformer in _model.Transformers) + { + var vm = TransformerFactory.Create(transformer, () => + { + SyncTransformersToModel(); + _onChanged(); + }); + if (vm != null) + Transformers.Add(vm); + } + } + + // Destination + Destination = new DestinationStepViewModel(_model.Destination, () => + { + _onChanged(); + }); + + // Post-scripts + PostScripts.Clear(); + if (_model.PostScripts != null) + { + foreach (var script in _model.PostScripts) + { + PostScripts.Add(new PostScriptStepViewModel(script, () => + { + SyncPostScriptsToModel(); + _onChanged(); + })); + } + } + + OnPropertyChanged(nameof(AllSteps)); + } + + private void UpdateSelectedStepEditor() + { + // The selected step IS the editor - we use the step VM directly + // The view will use DataTemplates to show the appropriate editor + SelectedStepEditor = _selectedStep; + } + + private void AddPreScript() + { + var step = new PreScriptStepViewModel(string.Empty, () => + { + SyncPreScriptsToModel(); + _onChanged(); + }); + PreScripts.Add(step); + SyncPreScriptsToModel(); + _onChanged(); + OnPropertyChanged(nameof(AllSteps)); + SelectedStep = step; + } + + private void AddTransformer() + { + // Default to ColumnDrop if nothing selected + var type = SelectedTransformerType ?? "ColumnDrop"; + var vm = TransformerFactory.CreateNew(type, () => + { + SyncTransformersToModel(); + _onChanged(); + }); + if (vm != null) + { + Transformers.Add(vm); + SyncTransformersToModel(); + _onChanged(); + OnPropertyChanged(nameof(AllSteps)); + SelectedStep = vm; + } + } + + /// + /// Adds a specific transformer type. + /// + public void AddTransformerOfType(string typeName) + { + var vm = TransformerFactory.CreateNew(typeName, () => + { + SyncTransformersToModel(); + _onChanged(); + }); + if (vm != null) + { + Transformers.Add(vm); + SyncTransformersToModel(); + _onChanged(); + OnPropertyChanged(nameof(AllSteps)); + SelectedStep = vm; + } + } + + private void AddPostScript() + { + var step = new PostScriptStepViewModel(string.Empty, () => + { + SyncPostScriptsToModel(); + _onChanged(); + }); + PostScripts.Add(step); + SyncPostScriptsToModel(); + _onChanged(); + OnPropertyChanged(nameof(AllSteps)); + SelectedStep = step; + } + + private void RemoveStep(PipelineStepViewModelBase? step) + { + if (step == null) return; + + switch (step) + { + case PreScriptStepViewModel preScript: + PreScripts.Remove(preScript); + SyncPreScriptsToModel(); + break; + case TransformerStepViewModelBase transformer: + Transformers.Remove(transformer); + SyncTransformersToModel(); + break; + case PostScriptStepViewModel postScript: + PostScripts.Remove(postScript); + SyncPostScriptsToModel(); + break; + // Source and Destination cannot be removed + } + + _onChanged(); + OnPropertyChanged(nameof(AllSteps)); + + if (SelectedStep == step) + SelectedStep = null; + } + + private bool CanMoveStepUp(PipelineStepViewModelBase? step) + { + if (step == null) return false; + + return step switch + { + PreScriptStepViewModel preScript => PreScripts.IndexOf(preScript) > 0, + TransformerStepViewModelBase transformer => Transformers.IndexOf(transformer) > 0, + PostScriptStepViewModel postScript => PostScripts.IndexOf(postScript) > 0, + _ => false + }; + } + + private void MoveStepUp(PipelineStepViewModelBase? step) + { + if (step == null || !CanMoveStepUp(step)) return; + + switch (step) + { + case PreScriptStepViewModel preScript: + MoveInCollection(PreScripts, preScript, -1); + SyncPreScriptsToModel(); + break; + case TransformerStepViewModelBase transformer: + MoveInCollection(Transformers, transformer, -1); + SyncTransformersToModel(); + break; + case PostScriptStepViewModel postScript: + MoveInCollection(PostScripts, postScript, -1); + SyncPostScriptsToModel(); + break; + } + + _onChanged(); + OnPropertyChanged(nameof(AllSteps)); + } + + private bool CanMoveStepDown(PipelineStepViewModelBase? step) + { + if (step == null) return false; + + return step switch + { + PreScriptStepViewModel preScript => PreScripts.IndexOf(preScript) < PreScripts.Count - 1, + TransformerStepViewModelBase transformer => Transformers.IndexOf(transformer) < Transformers.Count - 1, + PostScriptStepViewModel postScript => PostScripts.IndexOf(postScript) < PostScripts.Count - 1, + _ => false + }; + } + + private void MoveStepDown(PipelineStepViewModelBase? step) + { + if (step == null || !CanMoveStepDown(step)) return; + + switch (step) + { + case PreScriptStepViewModel preScript: + MoveInCollection(PreScripts, preScript, 1); + SyncPreScriptsToModel(); + break; + case TransformerStepViewModelBase transformer: + MoveInCollection(Transformers, transformer, 1); + SyncTransformersToModel(); + break; + case PostScriptStepViewModel postScript: + MoveInCollection(PostScripts, postScript, 1); + SyncPostScriptsToModel(); + break; + } + + _onChanged(); + OnPropertyChanged(nameof(AllSteps)); + } + + private static void MoveInCollection(ObservableCollection collection, T item, int offset) + { + var index = collection.IndexOf(item); + if (index < 0) return; + var newIndex = index + offset; + if (newIndex < 0 || newIndex >= collection.Count) return; + collection.Move(index, newIndex); + } + + private void SyncPreScriptsToModel() + { + _model.PreScripts = PreScripts.Count > 0 + ? PreScripts.Select(s => s.Script).ToArray() + : null; + } + + private void SyncTransformersToModel() + { + _model.Transformers = Transformers.Count > 0 + ? Transformers.Select(t => t.ToModel()).ToList() + : null; + } + + private void SyncPostScriptsToModel() + { + _model.PostScripts = PostScripts.Count > 0 + ? PostScripts.Select(s => s.Script).ToArray() + : null; + } +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/MainWindowViewModel.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/MainWindowViewModel.cs index 747474c..16d9087 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/MainWindowViewModel.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/MainWindowViewModel.cs @@ -183,6 +183,16 @@ public class MainWindowViewModel : ViewModelBase /// public ICommand DeleteSecretCommand { get; } + /// + /// Gets the command for adding a new pipeline. + /// + public ICommand AddPipelineCommand { get; } + + /// + /// Gets the command for deleting the selected pipeline. + /// + public ICommand DeletePipelineCommand { get; } + /// /// Initializes a new instance of the class. /// @@ -235,6 +245,10 @@ public class MainWindowViewModel : ViewModelBase AddSecretCommand = new AsyncRelayCommand(AddSecretAsync, CanAddSecret); DeleteSecretCommand = new AsyncRelayCommand(DeleteSecretAsync, CanDeleteSecret); + // Pipeline commands + AddPipelineCommand = new AsyncRelayCommand(AddPipelineAsync, CanAddPipeline); + DeletePipelineCommand = new AsyncRelayCommand(DeletePipelineAsync, CanDeletePipeline); + _ = InitializeAsync(); } @@ -271,9 +285,67 @@ public class MainWindowViewModel : ViewModelBase if (folder != null) { await LoadConfigAsync(folder); + await EnsureDefaultSecureStoreAsync(folder); } } + /// + /// Ensures a default secure store exists and is loaded. + /// Creates one if it doesn't exist. + /// + private async Task EnsureDefaultSecureStoreAsync(string configFolder) + { + var defaultStorePath = Path.Combine(configFolder, "default.secrets.json"); + var defaultKeyPath = Path.Combine(configFolder, "default.secrets.key"); + + try + { + // Create default store if it doesn't exist + if (!File.Exists(defaultStorePath)) + { + _logger?.LogInformation("Creating default secure store at {Path}", defaultStorePath); + + _secureStoreManager.CreateStore(defaultStorePath, defaultKeyPath); + + // Add some example secrets + _secureStoreManager.SetSecret("jde-password", ""); + _secureStoreManager.SetSecret("cms-password", ""); + _secureStoreManager.SetSecret("lotfinder-password", ""); + _secureStoreManager.Save(); + _secureStoreManager.CloseStore(); + + // Rebuild tree to show the new store + BuildTreeNodes(); + } + + // Auto-unlock the default store if key file exists + if (File.Exists(defaultStorePath) && File.Exists(defaultKeyPath)) + { + // Find the default store node in the tree + var secureStoresFolder = TreeNodes.FirstOrDefault(n => n.NodeType == TreeNodeType.SecureStoresFolder); + var defaultStoreNode = secureStoresFolder?.Children.FirstOrDefault(n => + n.StorePath != null && n.StorePath.EndsWith("default.secrets.json")); + + if (defaultStoreNode != null) + { + _secureStoreManager.OpenStore(defaultStorePath, defaultKeyPath); + defaultStoreNode.IsUnlocked = true; + _openStores[defaultStorePath] = defaultStoreNode; + RefreshStoreChildren(defaultStoreNode); + defaultStoreNode.IsExpanded = true; + + _logger?.LogInformation("Auto-unlocked default secure store"); + } + } + } + catch (Exception ex) + { + _logger?.LogWarning(ex, "Failed to initialize default secure store"); + } + + await Task.CompletedTask; + } + /// /// Opens a folder picker dialog to select a configuration folder. /// @@ -331,28 +403,28 @@ public class MainWindowViewModel : ViewModelBase TreeNodes.Clear(); // Settings folder - var settingsFolder = new TreeNodeViewModel("Settings", "gear", TreeNodeType.Folder) { IsExpanded = true }; - settingsFolder.Children.Add(new TreeNodeViewModel("DataSync", "sync", TreeNodeType.SettingsSection) { SectionKey = "DataSync" }); - settingsFolder.Children.Add(new TreeNodeViewModel("DataAccess", "database", TreeNodeType.SettingsSection) { SectionKey = "DataAccess" }); - settingsFolder.Children.Add(new TreeNodeViewModel("Auth", "lock", TreeNodeType.SettingsSection) { SectionKey = "Auth" }); - settingsFolder.Children.Add(new TreeNodeViewModel("Ldap", "users", TreeNodeType.SettingsSection) { SectionKey = "Ldap" }); - settingsFolder.Children.Add(new TreeNodeViewModel("Search", "search", TreeNodeType.SettingsSection) { SectionKey = "Search" }); - settingsFolder.Children.Add(new TreeNodeViewModel("ExcelExport", "file-spreadsheet", TreeNodeType.SettingsSection) { SectionKey = "ExcelExport" }); + var settingsFolder = new TreeNodeViewModel("Settings", "⚙️", TreeNodeType.Folder) { IsExpanded = true }; + settingsFolder.Children.Add(new TreeNodeViewModel("DataSync", "🔄", TreeNodeType.SettingsSection) { SectionKey = "DataSync" }); + settingsFolder.Children.Add(new TreeNodeViewModel("DataAccess", "🗄️", TreeNodeType.SettingsSection) { SectionKey = "DataAccess" }); + settingsFolder.Children.Add(new TreeNodeViewModel("Auth", "🔐", TreeNodeType.SettingsSection) { SectionKey = "Auth" }); + settingsFolder.Children.Add(new TreeNodeViewModel("Ldap", "👥", TreeNodeType.SettingsSection) { SectionKey = "Ldap" }); + settingsFolder.Children.Add(new TreeNodeViewModel("Search", "🔍", TreeNodeType.SettingsSection) { SectionKey = "Search" }); + settingsFolder.Children.Add(new TreeNodeViewModel("ExcelExport", "📊", TreeNodeType.SettingsSection) { SectionKey = "ExcelExport" }); TreeNodes.Add(settingsFolder); // Pipelines folder - var pipelinesFolder = new TreeNodeViewModel("Pipelines", "workflow", TreeNodeType.Folder) { IsExpanded = true }; + var pipelinesFolder = new TreeNodeViewModel("Pipelines", "⚡", TreeNodeType.Folder) { IsExpanded = true }; if (_pipelines != null) { foreach (var (name, _) in _pipelines.Pipelines) { - pipelinesFolder.Children.Add(new TreeNodeViewModel(name, "zap", TreeNodeType.Pipeline) { SectionKey = name }); + pipelinesFolder.Children.Add(new TreeNodeViewModel(name, "📦", TreeNodeType.Pipeline) { SectionKey = name }); } } TreeNodes.Add(pipelinesFolder); // Secure Stores folder - var secureStoresFolder = new TreeNodeViewModel("Secure Stores", "key", TreeNodeType.SecureStoresFolder) { IsExpanded = true }; + var secureStoresFolder = new TreeNodeViewModel("Secure Stores", "🔑", TreeNodeType.SecureStoresFolder) { IsExpanded = true }; DiscoverSecureStores(secureStoresFolder); TreeNodes.Add(secureStoresFolder); } @@ -378,7 +450,7 @@ public class MainWindowViewModel : ViewModelBase if (storeName.EndsWith(".secrets")) storeName = storeName[..^8]; // Remove ".secrets" suffix for display - var storeNode = new TreeNodeViewModel(storeName, "lock", TreeNodeType.SecureStore) + var storeNode = new TreeNodeViewModel(storeName, "🔒", TreeNodeType.SecureStore) { StorePath = filePath, SectionKey = filePath, @@ -400,6 +472,8 @@ public class MainWindowViewModel : ViewModelBase /// private void OnSelectedNodeChanged() { + RaisePipelineCommandsCanExecuteChanged(); + if (_selectedNode == null) { SelectedFormViewModel = null; @@ -455,7 +529,7 @@ public class MainWindowViewModel : ViewModelBase "ExcelExport" => new ExcelExportFormViewModel(_appSettings.ExcelExport, MarkAsChanged), _ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null => _pipelines.Pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline) - ? new PipelineFormViewModel(_selectedNode.SectionKey!, pipeline, MarkAsChanged) + ? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), MarkAsChanged) : null, _ => null }; @@ -682,6 +756,140 @@ public class MainWindowViewModel : ViewModelBase await Task.CompletedTask; } + /// + /// Gets the list of available connection names from the configuration. + /// + private IReadOnlyList GetAvailableConnections() + { + // Return well-known connection names for the JDE Scoping Tool + // These match the connection string names in appsettings.json + return new List + { + "jde", + "cms", + "giw", + "lotfinder" + }.AsReadOnly(); + } + + #region Pipeline Commands + + /// + /// Determines whether a new pipeline can be added. + /// + private bool CanAddPipeline() + { + return _pipelines != null; + } + + /// + /// Determines whether the selected pipeline can be deleted. + /// + private bool CanDeletePipeline() + { + return _selectedNode?.NodeType == TreeNodeType.Pipeline + && _pipelines != null; + } + + /// + /// Adds a new pipeline to the configuration. + /// + private async Task AddPipelineAsync() + { + if (_pipelines == null || _dialogService == null) + return; + + // Get pipeline name from user + var name = await _dialogService.ShowInputDialogAsync( + "New Pipeline", + "Enter pipeline name:"); + + if (string.IsNullOrWhiteSpace(name)) + return; + + // Check for duplicate + if (_pipelines.Pipelines.ContainsKey(name)) + { + await _dialogService.ShowMessageAsync("Error", + $"Pipeline '{name}' already exists."); + return; + } + + // Create default pipeline model + var pipeline = new PipelineModel + { + Source = new PipelineSource { Connection = "lotfinder", Query = "" }, + Destination = new PipelineDestination { Table = name }, + Schedules = new PipelineSchedules() + }; + + _pipelines.Pipelines[name] = pipeline; + + // Add tree node + var pipelinesFolder = TreeNodes.FirstOrDefault(n => + n.Name == "Pipelines" && n.NodeType == TreeNodeType.Folder); + if (pipelinesFolder != null) + { + var node = new TreeNodeViewModel(name, "📦", TreeNodeType.Pipeline) + { SectionKey = name }; + pipelinesFolder.Children.Add(node); + SelectedNode = node; + } + + MarkAsChanged(); + RaisePipelineCommandsCanExecuteChanged(); + + _logger?.LogInformation("Pipeline created: {Name}", name); + } + + /// + /// Deletes the selected pipeline from the configuration. + /// + private async Task DeletePipelineAsync() + { + if (_selectedNode?.NodeType != TreeNodeType.Pipeline || + _pipelines == null || + _dialogService == null) + return; + + var name = _selectedNode.SectionKey!; + + var confirmed = await _dialogService.ShowConfirmationAsync( + "Delete Pipeline", + $"Are you sure you want to delete pipeline '{name}'?"); + + if (!confirmed) + return; + + // Remove from model + _pipelines.Pipelines.Remove(name); + + // Remove tree node + var pipelinesFolder = TreeNodes.FirstOrDefault(n => + n.Name == "Pipelines" && n.NodeType == TreeNodeType.Folder); + pipelinesFolder?.Children.Remove(_selectedNode); + + // Clear selection + SelectedNode = pipelinesFolder; + SelectedFormViewModel = null; + + MarkAsChanged(); + RaisePipelineCommandsCanExecuteChanged(); + + _logger?.LogInformation("Pipeline deleted: {Name}", name); + } + + /// + /// Raises CanExecuteChanged for all Pipeline commands. + /// + private void RaisePipelineCommandsCanExecuteChanged() + { + (AddPipelineCommand as AsyncRelayCommand)?.RaiseCanExecuteChanged(); + (DeletePipelineCommand as AsyncRelayCommand)?.RaiseCanExecuteChanged(); + } + + #endregion + #region SecureStore Commands /// @@ -1014,7 +1222,7 @@ public class MainWindowViewModel : ViewModelBase var keys = _secureStoreManager.GetKeys(); foreach (var key in keys.OrderBy(k => k)) { - var secretNode = new TreeNodeViewModel(key, "key", TreeNodeType.Secret) + var secretNode = new TreeNodeViewModel(key, "🔐", TreeNodeType.Secret) { SecretKey = key, SectionKey = key diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/DestinationStepViewModel.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/DestinationStepViewModel.cs new file mode 100644 index 0000000..fb68299 --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/DestinationStepViewModel.cs @@ -0,0 +1,120 @@ +using JdeScoping.ConfigManager.Models; + +namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps; + +/// +/// Destination type for the pipeline. +/// +public enum DestinationType +{ + BulkImport, + BulkMerge +} + +/// +/// View model for the destination step in a pipeline. +/// +public class DestinationStepViewModel : PipelineStepViewModelBase +{ + private readonly PipelineDestination _model; + + public DestinationStepViewModel(PipelineDestination model, Action onChanged) : base(onChanged) + { + _model = model ?? throw new ArgumentNullException(nameof(model)); + } + + public override PipelineStepType StepType => PipelineStepType.Destination; + public override string DisplayName => "Destination"; + public override string Icon => "󰆼"; // mdi-database + public override string Summary => !string.IsNullOrEmpty(Table) ? $"→ {Table}" : "(no table)"; + + /// + /// Gets or sets the destination type (BulkImport or BulkMerge). + /// + public DestinationType Type + { + get => _model.Type?.Equals("BulkImport", StringComparison.OrdinalIgnoreCase) == true + ? DestinationType.BulkImport + : DestinationType.BulkMerge; + set + { + var typeStr = value == DestinationType.BulkImport ? "BulkImport" : "BulkMerge"; + if (_model.Type != typeStr) + { + _model.Type = typeStr; + OnPropertyChanged(); + OnPropertyChanged(nameof(IsBulkMerge)); + OnPropertyChanged(nameof(TypeDescription)); + NotifyChanged(); + } + } + } + + /// + /// Gets whether the destination type is BulkMerge (shows match columns). + /// + public bool IsBulkMerge => Type == DestinationType.BulkMerge; + + /// + /// Gets a description of the current type. + /// + public string TypeDescription => Type == DestinationType.BulkImport + ? "Truncate table and bulk load all data" + : "Merge data using match columns (upsert)"; + + /// + /// Gets or sets the destination table name. + /// + public string Table + { + get => _model.Table; + set + { + if (_model.Table != value) + { + _model.Table = value ?? string.Empty; + OnPropertyChanged(); + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + /// + /// Gets or sets the match columns as newline-separated text. + /// Only used for BulkMerge type. + /// + public string MatchColumnsText + { + get => string.Join("\n", _model.MatchColumns); + set + { + var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (!_model.MatchColumns.SequenceEqual(columns)) + { + _model.MatchColumns = columns; + OnPropertyChanged(); + NotifyChanged(); + } + } + } + + /// + /// Gets or sets the columns to exclude from updates as newline-separated text. + /// Only used for BulkMerge type. + /// + public string ExcludeFromUpdateText + { + get => string.Join("\n", _model.ExcludeFromUpdate); + set + { + var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (!_model.ExcludeFromUpdate.SequenceEqual(columns)) + { + _model.ExcludeFromUpdate = columns; + OnPropertyChanged(); + NotifyChanged(); + } + } + } +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/PipelineStepViewModelBase.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/PipelineStepViewModelBase.cs new file mode 100644 index 0000000..7fd660b --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/PipelineStepViewModelBase.cs @@ -0,0 +1,148 @@ +using System.Windows.Input; + +namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps; + +/// +/// Type of pipeline step in the flow diagram. +/// +public enum PipelineStepType +{ + PreScript, + Source, + Transformer, + Destination, + PostScript +} + +/// +/// Base class for all pipeline step view models in the visual flow diagram. +/// +public abstract class PipelineStepViewModelBase : ViewModelBase +{ + private bool _isSelected; + private readonly Action _onChanged; + + protected PipelineStepViewModelBase(Action onChanged) + { + _onChanged = onChanged ?? throw new ArgumentNullException(nameof(onChanged)); + } + + /// + /// Gets the type of this pipeline step. + /// + public abstract PipelineStepType StepType { get; } + + /// + /// Gets the display name for this step. + /// + public abstract string DisplayName { get; } + + /// + /// Gets the icon character for this step (using Material Design Icons). + /// + public abstract string Icon { get; } + + /// + /// Gets a short description for this step shown in the flow diagram. + /// + public abstract string Summary { get; } + + /// + /// Gets or sets whether this step is currently selected. + /// + public bool IsSelected + { + get => _isSelected; + set => SetProperty(ref _isSelected, value); + } + + /// + /// Notifies that the step has changed. + /// + protected void NotifyChanged() + { + _onChanged(); + } +} + +/// +/// View model for a pre-script step. +/// +public class PreScriptStepViewModel : PipelineStepViewModelBase +{ + private string _script; + + public PreScriptStepViewModel(string script, Action onChanged) : base(onChanged) + { + _script = script ?? string.Empty; + } + + public override PipelineStepType StepType => PipelineStepType.PreScript; + public override string DisplayName => "Pre-Script"; + public override string Icon => "󰯂"; // mdi-script-text + public override string Summary => TruncateScript(_script); + + /// + /// Gets or sets the SQL script content. + /// + public string Script + { + get => _script; + set + { + if (SetProperty(ref _script, value ?? string.Empty)) + { + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + private static string TruncateScript(string script) + { + if (string.IsNullOrWhiteSpace(script)) return "(empty)"; + var firstLine = script.Split('\n')[0].Trim(); + return firstLine.Length > 30 ? firstLine[..27] + "..." : firstLine; + } +} + +/// +/// View model for a post-script step. +/// +public class PostScriptStepViewModel : PipelineStepViewModelBase +{ + private string _script; + + public PostScriptStepViewModel(string script, Action onChanged) : base(onChanged) + { + _script = script ?? string.Empty; + } + + public override PipelineStepType StepType => PipelineStepType.PostScript; + public override string DisplayName => "Post-Script"; + public override string Icon => "󰯂"; // mdi-script-text + public override string Summary => TruncateScript(_script); + + /// + /// Gets or sets the SQL script content. + /// + public string Script + { + get => _script; + set + { + if (SetProperty(ref _script, value ?? string.Empty)) + { + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + private static string TruncateScript(string script) + { + if (string.IsNullOrWhiteSpace(script)) return "(empty)"; + var firstLine = script.Split('\n')[0].Trim(); + return firstLine.Length > 30 ? firstLine[..27] + "..." : firstLine; + } +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/SourceStepViewModel.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/SourceStepViewModel.cs new file mode 100644 index 0000000..a69aca9 --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/SourceStepViewModel.cs @@ -0,0 +1,298 @@ +using JdeScoping.ConfigManager.Models; +using System.Collections.ObjectModel; +using System.Windows.Input; + +namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps; + +/// +/// Source type for the pipeline. +/// +public enum SourceType +{ + Database, + File +} + +/// +/// View model for the source step in a pipeline. +/// +public class SourceStepViewModel : PipelineStepViewModelBase +{ + private readonly PipelineSource _model; + + public SourceStepViewModel(PipelineSource model, Action onChanged) : base(onChanged) + { + _model = model ?? throw new ArgumentNullException(nameof(model)); + + // Initialize parameters collection + Parameters = new ObservableCollection( + _model.Parameters.Select(kvp => new ParameterViewModel(kvp.Key, kvp.Value, () => + { + SyncParametersToModel(); + NotifyChanged(); + }))); + + // Initialize commands + AddParameterCommand = new RelayCommand(AddParameter); + } + + public override PipelineStepType StepType => PipelineStepType.Source; + public override string DisplayName => "Source"; + public override string Icon => IsFileSource ? "󰈔" : "󰆼"; // mdi-file vs mdi-database + public override string Summary => IsFileSource + ? $"File: {System.IO.Path.GetFileName(FileName) ?? "(none)"}" + : $"{Connection}: {TruncateQuery(Query)}"; + + /// + /// Gets whether this is a file-based source. + /// + public bool IsFileSource => !string.IsNullOrEmpty(_model.FileName); + + /// + /// Gets whether this is a database source. + /// + public bool IsDatabaseSource => !IsFileSource; + + /// + /// Gets or sets the source type (Database or File). + /// + public SourceType SourceType + { + get => IsFileSource ? SourceType.File : SourceType.Database; + set + { + if (value == SourceType.File && !IsFileSource) + { + // Switching to file source + _model.FileName = string.Empty; + _model.Connection = string.Empty; + _model.Query = string.Empty; + _model.MassQuery = null; + OnPropertyChanged(); + OnPropertyChanged(nameof(IsFileSource)); + OnPropertyChanged(nameof(IsDatabaseSource)); + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + else if (value == SourceType.Database && IsFileSource) + { + // Switching to database source + _model.FileName = null; + OnPropertyChanged(); + OnPropertyChanged(nameof(IsFileSource)); + OnPropertyChanged(nameof(IsDatabaseSource)); + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + /// + /// Gets or sets the source database connection name. + /// + public string Connection + { + get => _model.Connection; + set + { + if (_model.Connection != value) + { + _model.Connection = value ?? string.Empty; + OnPropertyChanged(); + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + /// + /// Gets or sets the query to extract data from the source. + /// + public string Query + { + get => _model.Query; + set + { + if (_model.Query != value) + { + _model.Query = value ?? string.Empty; + OnPropertyChanged(); + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + /// + /// Gets or sets the optional mass query for full data extraction. + /// + public string? MassQuery + { + get => _model.MassQuery; + set + { + if (_model.MassQuery != value) + { + _model.MassQuery = value; + OnPropertyChanged(); + NotifyChanged(); + } + } + } + + /// + /// Gets or sets the file name for file-based sources. + /// + public string? FileName + { + get => _model.FileName; + set + { + if (_model.FileName != value) + { + _model.FileName = value; + OnPropertyChanged(); + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + /// + /// Gets the collection of query parameters. + /// + public ObservableCollection Parameters { get; } + + /// + /// Gets the command to add a new parameter. + /// + public ICommand AddParameterCommand { get; } + + /// + /// Adds a new parameter. + /// + public void AddParameter() + { + var key = $"param{Parameters.Count + 1}"; + var param = new ParameterDefinition { Name = key }; + var vm = new ParameterViewModel(key, param, () => + { + SyncParametersToModel(); + NotifyChanged(); + }); + Parameters.Add(vm); + SyncParametersToModel(); + NotifyChanged(); + } + + /// + /// Removes a parameter. + /// + public void RemoveParameter(ParameterViewModel parameter) + { + if (Parameters.Remove(parameter)) + { + SyncParametersToModel(); + NotifyChanged(); + } + } + + private void SyncParametersToModel() + { + _model.Parameters.Clear(); + foreach (var p in Parameters) + { + _model.Parameters[p.Key] = p.ToModel(); + } + } + + private static string TruncateQuery(string query) + { + if (string.IsNullOrWhiteSpace(query)) return "(no query)"; + var trimmed = query.Trim().Replace("\n", " ").Replace("\r", ""); + return trimmed.Length > 25 ? trimmed[..22] + "..." : trimmed; + } +} + +/// +/// View model for a query parameter. +/// +public class ParameterViewModel : ViewModelBase +{ + private string _key; + private string _name; + private string? _format; + private string? _source; + private readonly Action _onChanged; + + public ParameterViewModel(string key, ParameterDefinition model, Action onChanged) + { + _key = key; + _name = model.Name; + _format = model.Format; + _source = model.Source; + _onChanged = onChanged; + } + + /// + /// Gets or sets the parameter key (used in the dictionary). + /// + public string Key + { + get => _key; + set + { + if (SetProperty(ref _key, value ?? string.Empty)) + _onChanged(); + } + } + + /// + /// Gets or sets the parameter name. + /// + public string Name + { + get => _name; + set + { + if (SetProperty(ref _name, value ?? string.Empty)) + _onChanged(); + } + } + + /// + /// Gets or sets the parameter format (e.g., jdeJulian, jdeTime). + /// + public string? Format + { + get => _format; + set + { + if (SetProperty(ref _format, value)) + _onChanged(); + } + } + + /// + /// Gets or sets the parameter source (e.g., offset, static). + /// + public string? Source + { + get => _source; + set + { + if (SetProperty(ref _source, value)) + _onChanged(); + } + } + + /// + /// Converts this view model back to a model. + /// + public ParameterDefinition ToModel() => new() + { + Name = _name, + Format = _format, + Source = _source + }; +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/TransformerStepViewModels.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/TransformerStepViewModels.cs new file mode 100644 index 0000000..9990483 --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/PipelineSteps/TransformerStepViewModels.cs @@ -0,0 +1,318 @@ +using JdeScoping.ConfigManager.Models; +using System.Collections.ObjectModel; +using System.Windows.Input; + +namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps; + +/// +/// Base class for transformer step view models. +/// +public abstract class TransformerStepViewModelBase : PipelineStepViewModelBase +{ + protected TransformerStepViewModelBase(Action onChanged) : base(onChanged) + { + } + + public override PipelineStepType StepType => PipelineStepType.Transformer; + public override string Icon => "󰁖"; // mdi-cog-transfer + + /// + /// Gets the transformer type name. + /// + public abstract string TransformerType { get; } + + /// + /// Converts this view model back to a model. + /// + public abstract TransformerModel ToModel(); +} + +/// +/// View model for ColumnDrop transformer. +/// +public class ColumnDropTransformerViewModel : TransformerStepViewModelBase +{ + private string _columnsText; + + public ColumnDropTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged) + { + _columnsText = model.Columns != null ? string.Join("\n", model.Columns) : string.Empty; + } + + public ColumnDropTransformerViewModel(Action onChanged) : base(onChanged) + { + _columnsText = string.Empty; + } + + public override string TransformerType => "ColumnDrop"; + public override string DisplayName => "Column Drop"; + public override string Summary => GetColumnCount() > 0 ? $"Drop {GetColumnCount()} columns" : "No columns"; + + /// + /// Gets or sets the columns to drop as newline-separated text. + /// + public string ColumnsText + { + get => _columnsText; + set + { + if (SetProperty(ref _columnsText, value ?? string.Empty)) + { + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + /// + /// Gets the columns as a list. + /// + public List GetColumns() + { + return _columnsText.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); + } + + private int GetColumnCount() + { + return _columnsText.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Length; + } + + public override TransformerModel ToModel() => new() + { + Type = TransformerType, + Columns = GetColumns() + }; +} + +/// +/// View model for ColumnRename transformer. +/// +public class ColumnRenameTransformerViewModel : TransformerStepViewModelBase +{ + public ColumnRenameTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged) + { + Mappings = new ObservableCollection( + model.Mappings?.Select(kvp => new ColumnMappingViewModel(kvp.Key, kvp.Value, () => + { + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + })) ?? []); + AddMappingCommand = new RelayCommand(AddMapping); + } + + public ColumnRenameTransformerViewModel(Action onChanged) : base(onChanged) + { + Mappings = []; + AddMappingCommand = new RelayCommand(AddMapping); + } + + public override string TransformerType => "ColumnRename"; + public override string DisplayName => "Column Rename"; + public override string Summary => Mappings.Count > 0 ? $"Rename {Mappings.Count} columns" : "No mappings"; + + /// + /// Gets the collection of column mappings (old name -> new name). + /// + public ObservableCollection Mappings { get; } + + /// + /// Gets the command to add a new mapping. + /// + public ICommand AddMappingCommand { get; } + + /// + /// Adds a new mapping. + /// + public void AddMapping() + { + Mappings.Add(new ColumnMappingViewModel("", "", () => + { + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + })); + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + + /// + /// Removes a mapping. + /// + public void RemoveMapping(ColumnMappingViewModel mapping) + { + if (Mappings.Remove(mapping)) + { + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + + public override TransformerModel ToModel() => new() + { + Type = TransformerType, + Mappings = Mappings.ToDictionary(m => m.OldName, m => m.NewName) + }; +} + +/// +/// View model for a column rename mapping. +/// +public class ColumnMappingViewModel : ViewModelBase +{ + private string _oldName; + private string _newName; + private readonly Action _onChanged; + + public ColumnMappingViewModel(string oldName, string newName, Action onChanged) + { + _oldName = oldName; + _newName = newName; + _onChanged = onChanged; + } + + /// + /// Gets or sets the original column name. + /// + public string OldName + { + get => _oldName; + set + { + if (SetProperty(ref _oldName, value ?? string.Empty)) + _onChanged(); + } + } + + /// + /// Gets or sets the new column name. + /// + public string NewName + { + get => _newName; + set + { + if (SetProperty(ref _newName, value ?? string.Empty)) + _onChanged(); + } + } +} + +/// +/// View model for JdeDate transformer (converts JDE Julian date/time to DateTime). +/// +public class JdeDateTransformerViewModel : TransformerStepViewModelBase +{ + private string? _dateColumn; + private string? _timeColumn; + private string? _outputColumn; + + public JdeDateTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged) + { + _dateColumn = model.DateColumn; + _timeColumn = model.TimeColumn; + _outputColumn = model.OutputColumn; + } + + public JdeDateTransformerViewModel(Action onChanged) : base(onChanged) + { + _dateColumn = null; + _timeColumn = null; + _outputColumn = null; + } + + public override string TransformerType => "JdeDate"; + public override string DisplayName => "JDE Date Convert"; + public override string Icon => "󰃭"; // mdi-calendar + public override string Summary => !string.IsNullOrEmpty(_outputColumn) ? $"→ {_outputColumn}" : "Configure..."; + + /// + /// Gets or sets the date column name. + /// + public string? DateColumn + { + get => _dateColumn; + set + { + if (SetProperty(ref _dateColumn, value)) + { + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + /// + /// Gets or sets the time column name. + /// + public string? TimeColumn + { + get => _timeColumn; + set + { + if (SetProperty(ref _timeColumn, value)) + NotifyChanged(); + } + } + + /// + /// Gets or sets the output column name. + /// + public string? OutputColumn + { + get => _outputColumn; + set + { + if (SetProperty(ref _outputColumn, value)) + { + OnPropertyChanged(nameof(Summary)); + NotifyChanged(); + } + } + } + + public override TransformerModel ToModel() => new() + { + Type = TransformerType, + DateColumn = _dateColumn, + TimeColumn = _timeColumn, + OutputColumn = _outputColumn + }; +} + +/// +/// Factory methods for creating transformer view models. +/// +public static class TransformerFactory +{ + /// + /// Creates a transformer view model from a model. + /// + public static TransformerStepViewModelBase? Create(TransformerModel model, Action onChanged) + { + return model.Type?.ToLowerInvariant() switch + { + "columndrop" => new ColumnDropTransformerViewModel(model, onChanged), + "columnrename" => new ColumnRenameTransformerViewModel(model, onChanged), + "jdedate" => new JdeDateTransformerViewModel(model, onChanged), + _ => null // Unknown transformer type + }; + } + + /// + /// Creates a new transformer view model by type name. + /// + public static TransformerStepViewModelBase? CreateNew(string typeName, Action onChanged) + { + return typeName?.ToLowerInvariant() switch + { + "columndrop" => new ColumnDropTransformerViewModel(onChanged), + "columnrename" => new ColumnRenameTransformerViewModel(onChanged), + "jdedate" => new JdeDateTransformerViewModel(onChanged), + _ => null + }; + } + + /// + /// Gets the list of available transformer type names. + /// + public static IReadOnlyList AvailableTypes => ["ColumnDrop", "ColumnRename", "JdeDate"]; +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/RelayCommand.cs b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/RelayCommand.cs index 2b1ef7f..d4be873 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/RelayCommand.cs +++ b/NEW/src/Utils/JdeScoping.ConfigManager/ViewModels/RelayCommand.cs @@ -60,3 +60,68 @@ public class RelayCommand : ICommand /// public void RaiseCanExecuteChanged() => _canExecuteChanged?.Invoke(this, EventArgs.Empty); } + +/// +/// A strongly-typed command implementation that delegates to action methods. +/// +/// The type of the command parameter. +public class RelayCommand : ICommand +{ + private readonly Action _execute; + private readonly Predicate? _canExecute; + private EventHandler? _canExecuteChanged; + + /// + /// Occurs when the result of has changed. + /// + public event EventHandler? CanExecuteChanged + { + add => _canExecuteChanged += value; + remove => _canExecuteChanged -= value; + } + + /// + /// Initializes a new instance of the class. + /// + /// The action to execute when the command is invoked. + /// An optional predicate to determine if the command can execute. + /// Thrown when is null. + public RelayCommand(Action execute, Predicate? canExecute = null) + { + _execute = execute ?? throw new ArgumentNullException(nameof(execute)); + _canExecute = canExecute; + } + + /// + /// Determines whether the command can execute in its current state. + /// + /// The parameter to pass to the canExecute predicate. + /// True if the command can execute; otherwise, false. + public bool CanExecute(object? parameter) + { + if (parameter is T typedParam) + return _canExecute?.Invoke(typedParam) ?? true; + if (parameter is null && !typeof(T).IsValueType) + return _canExecute?.Invoke(default) ?? true; + return _canExecute?.Invoke(default) ?? true; + } + + /// + /// Executes the command with the specified parameter. + /// + /// The parameter to pass to the execute action. + public void Execute(object? parameter) + { + if (parameter is T typedParam) + _execute(typedParam); + else if (parameter is null && !typeof(T).IsValueType) + _execute(default); + else + _execute(default); + } + + /// + /// Raises the event to notify command bindings of state changes. + /// + public void RaiseCanExecuteChanged() => _canExecuteChanged?.Invoke(this, EventArgs.Empty); +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/FlowArrow.axaml b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/FlowArrow.axaml new file mode 100644 index 0000000..5015464 --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/FlowArrow.axaml @@ -0,0 +1,16 @@ + + + + + + + + + + diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/FlowArrow.axaml.cs b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/FlowArrow.axaml.cs new file mode 100644 index 0000000..b6a06ea --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/FlowArrow.axaml.cs @@ -0,0 +1,11 @@ +using Avalonia.Controls; + +namespace JdeScoping.ConfigManager.Views.Controls; + +public partial class FlowArrow : UserControl +{ + public FlowArrow() + { + InitializeComponent(); + } +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/PipelineStepCard.axaml b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/PipelineStepCard.axaml new file mode 100644 index 0000000..82ada41 --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/PipelineStepCard.axaml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/PipelineStepCard.axaml.cs b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/PipelineStepCard.axaml.cs new file mode 100644 index 0000000..5353ed8 --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Controls/PipelineStepCard.axaml.cs @@ -0,0 +1,106 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Input; +using Avalonia.Media; +using JdeScoping.ConfigManager.ViewModels.Forms; +using JdeScoping.ConfigManager.ViewModels.PipelineSteps; + +namespace JdeScoping.ConfigManager.Views.Controls; + +public partial class PipelineStepCard : UserControl +{ + public static readonly StyledProperty StepColorProperty = + AvaloniaProperty.Register(nameof(StepColor), "#3B82F6"); + + public PipelineStepCard() + { + InitializeComponent(); + PointerPressed += OnPointerPressed; + PropertyChanged += OnPropertyChangedHandler; + DataContextChanged += OnDataContextChanged; + } + + /// + /// Gets or sets the step color for the icon background. + /// + public string StepColor + { + get => GetValue(StepColorProperty); + set => SetValue(StepColorProperty, value); + } + + private void OnPropertyChangedHandler(object? sender, AvaloniaPropertyChangedEventArgs e) + { + if (e.Property == StepColorProperty) + { + UpdateIconColor(); + } + } + + private void OnDataContextChanged(object? sender, EventArgs e) + { + UpdateIconColor(); + UpdateSelectionState(); + + if (DataContext is PipelineStepViewModelBase vm) + { + vm.PropertyChanged += (s, e) => + { + if (e.PropertyName == nameof(PipelineStepViewModelBase.IsSelected)) + { + UpdateSelectionState(); + } + }; + } + } + + private void UpdateIconColor() + { + var iconBg = this.FindControl("IconBackground"); + if (iconBg != null && !string.IsNullOrEmpty(StepColor)) + { + if (Color.TryParse(StepColor, out var color)) + { + iconBg.Background = new SolidColorBrush(color); + } + } + } + + private void UpdateSelectionState() + { + var cardBorder = this.FindControl("CardBorder"); + if (cardBorder != null && DataContext is PipelineStepViewModelBase vm) + { + if (vm.IsSelected) + { + cardBorder.BorderBrush = new SolidColorBrush(Color.Parse("#3B82F6")); + cardBorder.Background = new SolidColorBrush(Color.Parse("#1E2A3A")); + } + else + { + cardBorder.BorderBrush = new SolidColorBrush(Color.Parse("#3D4550")); + cardBorder.Background = new SolidColorBrush(Color.Parse("#1A1F26")); + } + } + } + + private void OnPointerPressed(object? sender, PointerPressedEventArgs e) + { + // Find the PipelineEditorViewModel from the visual tree + var parent = this.Parent; + while (parent != null) + { + if (parent is UserControl uc && uc.DataContext is PipelineEditorViewModel editorVm) + { + if (DataContext is PipelineStepViewModelBase stepVm) + { + editorVm.SelectedStep = stepVm; + } + break; + } + parent = parent.Parent; + } + + e.Handled = true; + } +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Dialogs/InputDialog.axaml b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Dialogs/InputDialog.axaml new file mode 100644 index 0000000..a6c7a2b --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Dialogs/InputDialog.axaml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/PipelineEditorView.axaml.cs b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/PipelineEditorView.axaml.cs new file mode 100644 index 0000000..471f0e7 --- /dev/null +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/Forms/PipelineEditorView.axaml.cs @@ -0,0 +1,11 @@ +using Avalonia.Controls; + +namespace JdeScoping.ConfigManager.Views.Forms; + +public partial class PipelineEditorView : UserControl +{ + public PipelineEditorView() + { + InitializeComponent(); + } +} diff --git a/NEW/src/Utils/JdeScoping.ConfigManager/Views/MainWindow.axaml b/NEW/src/Utils/JdeScoping.ConfigManager/Views/MainWindow.axaml index 918dbbf..7396d22 100644 --- a/NEW/src/Utils/JdeScoping.ConfigManager/Views/MainWindow.axaml +++ b/NEW/src/Utils/JdeScoping.ConfigManager/Views/MainWindow.axaml @@ -2,7 +2,9 @@ xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:vm="using:JdeScoping.ConfigManager.ViewModels" xmlns:forms="using:JdeScoping.ConfigManager.ViewModels.Forms" + xmlns:steps="using:JdeScoping.ConfigManager.ViewModels.PipelineSteps" xmlns:views="using:JdeScoping.ConfigManager.Views.Forms" + xmlns:editors="using:JdeScoping.ConfigManager.Views.Editors" x:Class="JdeScoping.ConfigManager.Views.MainWindow" x:DataType="vm:MainWindowViewModel" Title="JdeScoping ConfigManager" @@ -15,6 +17,7 @@ + @@ -33,9 +36,36 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -66,6 +96,18 @@ + + + + + + + + + + + + @@ -113,6 +155,8 @@