dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(datasync): correct MisData postScript and query filtering

Browse Source 
- Replace placeholder postScript with actual MIS data post-processing:
  1. Sets ObsoleteDate based on BackLevel records
  2. Sets ObsoleteDate for remaining NULL cases
  3. Rebuilds PK_MisData index

- Add massQuery support to SourceConfig for mode-specific queries
- MisData mass sync now uses query without date filter (like legacy)
- EtlPipelineFactory selects massQuery when in mass mode if available
- Remove unnecessary minDtOffset from MisData mass mode config
This commit is contained in:
Joseph Doherty
2026-01-06 14:01:26 -05:00
parent 4298fb8147
commit e75cd70d94
3 changed files with 20 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/JdeScoping.DataSync/Services/EtlPipelineFactory.cs
+13 -4
View File
@@ -192,7 +192,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
var minDt = _minDtOverride ?? ComputeMinDt(modeConfig.MinDtOffset);
// Create source with parameter substitution
var source = CreateSource(_config.Source, minDt);
var source = CreateSource(_config.Source, minDt, _mode);
// Determine destination type (mode override > default by mode)
var destType = modeConfig.Destination?.Type
@@ -251,12 +251,21 @@ public class EtlPipelineFactory : IEtlPipelineFactory
return DateTime.UtcNow.Add(offset);
}
private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt)
private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt, SyncMode mode)
{
// Use massQuery if available and in mass mode, otherwise use the default query
var query = (mode == SyncMode.Mass && !string.IsNullOrEmpty(sourceConfig.MassQuery))
? sourceConfig.MassQuery
: sourceConfig.Query;
var parameters = new Dictionary<string, object>();
var converter = new ParameterFormatConverter(_settings.Timezone);
if (sourceConfig.Parameters != null && minDt.HasValue)
// Only add parameters for incremental mode or when using the default query
// Mass mode with massQuery typically doesn't need date parameters
var needsParameters = mode != SyncMode.Mass || string.IsNullOrEmpty(sourceConfig.MassQuery);
if (sourceConfig.Parameters != null && minDt.HasValue && needsParameters)
{
foreach (var (_, paramConfig) in sourceConfig.Parameters)
{
@@ -278,7 +287,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
return new DbQuerySource(
_connectionFactory,
sourceConfig.Connection,
sourceConfig.Query,
query,
parameters);
}