From e75cd70d94545da404fbb8b6cf69a83c1f07ec68 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 6 Jan 2026 14:01:26 -0500 Subject: [PATCH] fix(datasync): correct MisData postScript and query filtering - Replace placeholder postScript with actual MIS data post-processing: 1. Sets ObsoleteDate based on BackLevel records 2. Sets ObsoleteDate for remaining NULL cases 3. Rebuilds PK_MisData index - Add massQuery support to SourceConfig for mode-specific queries - MisData mass sync now uses query without date filter (like legacy) - EtlPipelineFactory selects massQuery when in mass mode if available - Remove unnecessary minDtOffset from MisData mass mode config --- .../Configuration/PipelineConfig.cs | 3 ++- .../Pipelines/pipelines.json | 7 +++++-- .../Services/EtlPipelineFactory.cs | 17 +++++++++++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/NEW/src/JdeScoping.DataSync/Configuration/PipelineConfig.cs b/NEW/src/JdeScoping.DataSync/Configuration/PipelineConfig.cs index fa76e58..e4160df 100644 --- a/NEW/src/JdeScoping.DataSync/Configuration/PipelineConfig.cs +++ b/NEW/src/JdeScoping.DataSync/Configuration/PipelineConfig.cs @@ -11,7 +11,8 @@ public record PipelineConfig( public record SourceConfig( string Connection, string Query, - Dictionary? Parameters); + Dictionary? Parameters, + string? MassQuery = null); public record ParameterConfig( string Name, diff --git a/NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json b/NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json index 5144cc3..f0d607f 100644 --- a/NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json +++ b/NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json @@ -156,12 +156,13 @@ "source": { "connection": "cms", "query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME IN ('Current', 'BackLevel') AND Status.PDATE_RELEASED >= :lastUpdateDT", + "massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME IN ('Current', 'BackLevel')", "parameters": { "lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" } } }, "syncModes": { - "mass": { "minDtOffset": "-3650.00:00:00", "prePurge": true, "reIndex": true }, + "mass": { "prePurge": true, "reIndex": true }, "incremental": { "minDtOffset": "-7.00:00:00" } }, "destination": { @@ -169,7 +170,9 @@ "matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"] }, "postScripts": [ - "UPDATE MisData SET ProcessedFlag = 1 WHERE ProcessedFlag IS NULL" + "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData.MisNumber = bl.MisNumber AND MisData.RevID = bl.RevID AND MisData.Status = 'Current' AND bl.Status = 'BackLevel';", + "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData.MisNumber = nl.MisNumber AND MisData.RevID < nl.RevID AND MisData.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;", + "ALTER INDEX [PK_MisData] ON [dbo].[MisData] REBUILD;" ] } } diff --git a/NEW/src/JdeScoping.DataSync/Services/EtlPipelineFactory.cs b/NEW/src/JdeScoping.DataSync/Services/EtlPipelineFactory.cs index 00095c0..9ed61e6 100644 --- a/NEW/src/JdeScoping.DataSync/Services/EtlPipelineFactory.cs +++ b/NEW/src/JdeScoping.DataSync/Services/EtlPipelineFactory.cs @@ -192,7 +192,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory var minDt = _minDtOverride ?? ComputeMinDt(modeConfig.MinDtOffset); // Create source with parameter substitution - var source = CreateSource(_config.Source, minDt); + var source = CreateSource(_config.Source, minDt, _mode); // Determine destination type (mode override > default by mode) var destType = modeConfig.Destination?.Type @@ -251,12 +251,21 @@ public class EtlPipelineFactory : IEtlPipelineFactory return DateTime.UtcNow.Add(offset); } - private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt) + private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt, SyncMode mode) { + // Use massQuery if available and in mass mode, otherwise use the default query + var query = (mode == SyncMode.Mass && !string.IsNullOrEmpty(sourceConfig.MassQuery)) + ? sourceConfig.MassQuery + : sourceConfig.Query; + var parameters = new Dictionary(); var converter = new ParameterFormatConverter(_settings.Timezone); - if (sourceConfig.Parameters != null && minDt.HasValue) + // Only add parameters for incremental mode or when using the default query + // Mass mode with massQuery typically doesn't need date parameters + var needsParameters = mode != SyncMode.Mass || string.IsNullOrEmpty(sourceConfig.MassQuery); + + if (sourceConfig.Parameters != null && minDt.HasValue && needsParameters) { foreach (var (_, paramConfig) in sourceConfig.Parameters) { @@ -278,7 +287,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory return new DbQuerySource( _connectionFactory, sourceConfig.Connection, - sourceConfig.Query, + query, parameters); }