refactor(configmanager): migrate to per-file pipeline system
Align ConfigManager with DataSync's per-file pipeline format (pipeline.*.json) by reusing EtlPipelineConfig types directly, eliminating duplicate models and simplifying the codebase. Removes ~3200 lines of obsolete code.
This commit is contained in:
Joseph Doherty
@@ -1,34 +0,0 @@
|
||||
namespace JdeScoping.DataSync.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for an ETL pipeline.
|
||||
/// </summary>
|
||||
public record PipelineConfig(
|
||||
SourceConfig Source,
|
||||
PipelineSchedules? Schedules,
|
||||
List<TransformerConfig>? Transformers,
|
||||
DestinationConfig Destination,
|
||||
List<string>? PreScripts,
|
||||
List<string>? PostScripts);
|
||||
|
||||
public record SourceConfig(
|
||||
string Connection,
|
||||
string Query,
|
||||
Dictionary<string, ParameterConfig>? Parameters,
|
||||
string? MassQuery = null);
|
||||
|
||||
public record ParameterConfig(
|
||||
string Name,
|
||||
string? Format,
|
||||
string Source = "offset",
|
||||
string? Value = null);
|
||||
|
||||
public record TransformerConfig(
|
||||
string Type,
|
||||
List<string>? Columns,
|
||||
Dictionary<string, string>? Mappings);
|
||||
|
||||
public record DestinationConfig(
|
||||
string Table,
|
||||
List<string>? MatchColumns,
|
||||
List<string>? ExcludeFromUpdate);
|
||||
@@ -1,15 +0,0 @@
|
||||
namespace JdeScoping.DataSync.Configuration;
|
||||
|
||||
public record PipelinesRoot(
|
||||
PipelineSettings? Settings, // Optional - defaults applied if missing
|
||||
ScheduleDefaults? ScheduleDefaults, // Optional - defaults applied if missing
|
||||
Dictionary<string, PipelineConfig> Pipelines)
|
||||
{
|
||||
/// <summary>Gets the effective pipeline settings, using defaults if not specified.</summary>
|
||||
public PipelineSettings EffectiveSettings => Settings ?? new PipelineSettings();
|
||||
/// <summary>Gets the effective schedule defaults, using defaults if not specified.</summary>
|
||||
public ScheduleDefaults EffectiveScheduleDefaults => ScheduleDefaults ?? new ScheduleDefaults();
|
||||
}
|
||||
|
||||
public record PipelineSettings(
|
||||
string Timezone = "UTC");
|
||||
@@ -1,110 +0,0 @@
|
||||
namespace JdeScoping.DataSync.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a single schedule type (Mass/Daily/Hourly).
|
||||
/// </summary>
|
||||
public record ScheduleConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether this schedule is enabled.
|
||||
/// </summary>
|
||||
public bool Enabled { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Interval in minutes between syncs.
|
||||
/// </summary>
|
||||
public int IntervalMinutes { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to truncate the table before import (full reload).
|
||||
/// </summary>
|
||||
public bool PrePurge { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to rebuild indexes after import.
|
||||
/// </summary>
|
||||
public bool ReIndex { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Condition for updating existing rows (e.g., "src.LastUpdateDt > tgt.LastUpdateDt").
|
||||
/// </summary>
|
||||
public string? UpdateWhen { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Merges this config with defaults. Non-null/non-default values in this config override defaults.
|
||||
/// </summary>
|
||||
/// <param name="defaults">The default configuration to merge with.</param>
|
||||
public ScheduleConfig MergeWith(ScheduleConfig defaults)
|
||||
{
|
||||
return new ScheduleConfig
|
||||
{
|
||||
Enabled = Enabled,
|
||||
IntervalMinutes = IntervalMinutes > 0 ? IntervalMinutes : defaults.IntervalMinutes,
|
||||
PrePurge = PrePurge || defaults.PrePurge,
|
||||
ReIndex = ReIndex || defaults.ReIndex,
|
||||
UpdateWhen = UpdateWhen ?? defaults.UpdateWhen
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default schedule configurations for all pipelines.
|
||||
/// </summary>
|
||||
public record ScheduleDefaults
|
||||
{
|
||||
/// <summary>
|
||||
/// Default Mass schedule config (weekly, full reload).
|
||||
/// </summary>
|
||||
public ScheduleConfig Mass { get; init; } = new()
|
||||
{
|
||||
Enabled = true,
|
||||
IntervalMinutes = 10080, // Weekly
|
||||
PrePurge = true,
|
||||
ReIndex = true
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Default Daily schedule config (incremental merge).
|
||||
/// </summary>
|
||||
public ScheduleConfig Daily { get; init; } = new()
|
||||
{
|
||||
Enabled = true,
|
||||
IntervalMinutes = 1440, // Daily
|
||||
PrePurge = false,
|
||||
ReIndex = false,
|
||||
UpdateWhen = "src.LastUpdateDt > tgt.LastUpdateDt"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Default Hourly schedule config (incremental merge).
|
||||
/// </summary>
|
||||
public ScheduleConfig Hourly { get; init; } = new()
|
||||
{
|
||||
Enabled = true,
|
||||
IntervalMinutes = 60, // Hourly
|
||||
PrePurge = false,
|
||||
ReIndex = false,
|
||||
UpdateWhen = "src.LastUpdateDt > tgt.LastUpdateDt"
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-pipeline schedule overrides.
|
||||
/// </summary>
|
||||
public record PipelineSchedules
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or initializes the Mass schedule configuration override.
|
||||
/// </summary>
|
||||
public ScheduleConfig? Mass { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or initializes the Daily schedule configuration override.
|
||||
/// </summary>
|
||||
public ScheduleConfig? Daily { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or initializes the Hourly schedule configuration override.
|
||||
/// </summary>
|
||||
public ScheduleConfig? Hourly { get; init; }
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
|
||||
namespace JdeScoping.DataSync.Contracts;
|
||||
|
||||
public interface IEtlPipelineFactory
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a pipeline builder for the specified table.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The table name (pipeline key).</param>
|
||||
/// <returns>A builder for configuring the pipeline.</returns>
|
||||
IEtlPipelineBuilder ForTable(string tableName);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the list of available table names (pipeline keys).
|
||||
/// </summary>
|
||||
/// <returns>List of table names with configured pipelines.</returns>
|
||||
IReadOnlyList<string> GetAvailableTables();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configuration for a specific pipeline.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The table name (pipeline key).</param>
|
||||
/// <returns>The pipeline configuration, or null if not found.</returns>
|
||||
PipelineConfig? GetPipelineConfig(string tableName);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the schedule defaults from the configuration.
|
||||
/// </summary>
|
||||
/// <returns>The schedule defaults.</returns>
|
||||
ScheduleDefaults GetScheduleDefaults();
|
||||
}
|
||||
|
||||
public interface IEtlPipelineBuilder
|
||||
{
|
||||
/// <summary>
|
||||
/// Sets the update type for this pipeline (Mass, Daily, or Hourly).
|
||||
/// </summary>
|
||||
/// <param name="updateType">The update type.</param>
|
||||
/// <returns>The builder for chaining.</returns>
|
||||
IEtlPipelineBuilder WithUpdateType(UpdateTypes updateType);
|
||||
|
||||
/// <summary>
|
||||
/// Sets an optional minimum date for filtering source data.
|
||||
/// </summary>
|
||||
/// <param name="minDt">The minimum date, or null to use config offset.</param>
|
||||
/// <returns>The builder for chaining.</returns>
|
||||
IEtlPipelineBuilder WithMinimumDate(DateTime? minDt);
|
||||
|
||||
/// <summary>
|
||||
/// Builds the pipeline with the configured settings.
|
||||
/// </summary>
|
||||
/// <returns>The configured pipeline.</returns>
|
||||
EtlPipeline Build();
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
using JdeScoping.DataSync;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.HealthChecks;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using JdeScoping.DataSync.Telemetry;
|
||||
@@ -35,14 +35,10 @@ public static class DataSyncDependencyInjection
|
||||
.ValidateDataAnnotations()
|
||||
.ValidateOnStart();
|
||||
|
||||
// Pipeline configuration (new ETL infrastructure)
|
||||
services.AddOptions<PipelineOptions>()
|
||||
.Bind(configuration.GetSection(PipelineOptions.SectionName));
|
||||
// Pipeline builder service (builds ETL pipelines from config)
|
||||
services.AddSingleton<IEtlPipelineBuilder, EtlPipelineBuilderService>();
|
||||
|
||||
// Pipeline factory (new ETL infrastructure)
|
||||
services.AddSingleton<IEtlPipelineFactory, EtlPipelineFactory>();
|
||||
|
||||
// Pipeline registry services (new hot-reload infrastructure)
|
||||
// Pipeline registry services (hot-reload infrastructure)
|
||||
services.AddSingleton<IPipelineValidator, PipelineValidator>();
|
||||
services.AddSingleton<IPipelineRegistry, PipelineRegistry>();
|
||||
|
||||
|
||||
@@ -27,10 +27,4 @@
|
||||
<PackageReference Include="ZstdSharp.Port" Version="0.8.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="Pipelines\pipelines.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Options;
|
||||
|
||||
namespace JdeScoping.DataSync.Models;
|
||||
|
||||
@@ -41,16 +40,10 @@ public class DataUpdateTask
|
||||
public DateTime? MinimumDt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The pipeline configuration for this task (new format).
|
||||
/// The pipeline configuration for this task.
|
||||
/// </summary>
|
||||
public EtlPipelineConfig? Pipeline { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The data source configuration for this task (legacy format - will be removed).
|
||||
/// </summary>
|
||||
[Obsolete("Use Pipeline instead. This property exists for backward compatibility during migration.")]
|
||||
public DataSourceConfig? Config { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a unique key for logging purposes.
|
||||
/// </summary>
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
namespace JdeScoping.DataSync.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a single data source table sync.
|
||||
/// </summary>
|
||||
public class DataSourceConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Target table name in SQL Server cache.
|
||||
/// </summary>
|
||||
public required string TableName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Source system: "JDE" or "CMS".
|
||||
/// </summary>
|
||||
public required string SourceSystem { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Source data identifier (e.g., "WORKORDER", "LOTUSAGE").
|
||||
/// </summary>
|
||||
public string SourceData { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether this data source is enabled for sync.
|
||||
/// </summary>
|
||||
public bool IsEnabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Mass sync schedule configuration.
|
||||
/// </summary>
|
||||
public ScheduleConfig MassConfig { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Daily incremental sync configuration.
|
||||
/// </summary>
|
||||
public ScheduleConfig DailyConfig { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Hourly incremental sync configuration.
|
||||
/// </summary>
|
||||
public ScheduleConfig HourlyConfig { get; set; } = new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Schedule configuration for a sync type (Mass/Daily/Hourly).
|
||||
/// </summary>
|
||||
public class ScheduleConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether this schedule is enabled.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Interval in minutes between syncs.
|
||||
/// </summary>
|
||||
public int IntervalMinutes { get; set; }
|
||||
}
|
||||
@@ -69,9 +69,4 @@ public class DataSyncOptions
|
||||
/// If false, invalid enabled pipelines are skipped with warnings.
|
||||
/// </summary>
|
||||
public bool StrictPipelineValidation { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Per-table data source configurations.
|
||||
/// </summary>
|
||||
public List<DataSourceConfig> DataSources { get; set; } = [];
|
||||
}
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
namespace JdeScoping.DataSync.Options;
|
||||
|
||||
public class PipelineOptions
|
||||
{
|
||||
public const string SectionName = "Pipelines";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the path to the pipeline configuration file.
|
||||
/// </summary>
|
||||
public string ConfigPath { get; set; } = "Pipelines/pipelines.json";
|
||||
}
|
||||
@@ -1,392 +0,0 @@
|
||||
{
|
||||
"settings": {
|
||||
"timezone": "UTC"
|
||||
},
|
||||
"scheduleDefaults": {
|
||||
"mass": { "enabled": true, "intervalMinutes": 10080, "prePurge": true, "reIndex": true },
|
||||
"daily": { "enabled": true, "intervalMinutes": 1440, "prePurge": false, "reIndex": false },
|
||||
"hourly": { "enabled": true, "intervalMinutes": 60, "prePurge": false, "reIndex": false }
|
||||
},
|
||||
"pipelines": {
|
||||
"WorkOrder_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo WHERE (wo.WAUPMJ > :dateUpdated OR (wo.WAUPMJ = :dateUpdated AND wo.WATDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrder_Curr",
|
||||
"matchColumns": ["WorkOrderNumber", "BranchCode"],
|
||||
"excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"Lot": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL AND (lot.IOUPMJ > :dateUpdated OR (lot.IOUPMJ = :dateUpdated AND lot.IOTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "Lot",
|
||||
"matchColumns": ["LotNumber", "BranchCode"],
|
||||
"excludeFromUpdate": ["LotNumber", "BranchCode", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"LotUsage_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL AND (lu.ILTRDJ > :dateUpdated OR (lu.ILTRDJ = :dateUpdated AND lu.ILTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "LotUsage_Curr",
|
||||
"matchColumns": ["UniqueId"],
|
||||
"excludeFromUpdate": ["UniqueId", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"Item": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL AND (pn.IMUPMJ > :dateUpdated OR (pn.IMUPMJ = :dateUpdated AND pn.IMTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "Item",
|
||||
"matchColumns": ["ShortItemNumber"],
|
||||
"excludeFromUpdate": ["ShortItemNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"WorkCenter": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkCenter",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"ProfitCenter": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "ProfitCenter",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"JdeUser": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1",
|
||||
"massQuery": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1",
|
||||
"parameters": {}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "JdeUser",
|
||||
"matchColumns": ["AddressNumber"],
|
||||
"excludeFromUpdate": ["AddressNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"Branch": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "Branch",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"MisData_Curr": {
|
||||
"source": {
|
||||
"connection": "cms",
|
||||
"query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current' AND Status.PDATE_RELEASED >= :lastUpdateDT",
|
||||
"massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current'",
|
||||
"parameters": {
|
||||
"lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": { "intervalMinutes": 100800 },
|
||||
"daily": {},
|
||||
"hourly": { "enabled": false }
|
||||
},
|
||||
"destination": {
|
||||
"table": "MisData_Curr",
|
||||
"matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"]
|
||||
},
|
||||
"postScripts": [
|
||||
"SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Curr.MisNumber = bl.MisNumber AND MisData_Curr.RevID = bl.RevID AND MisData_Curr.Status = 'Current' AND bl.Status = 'BackLevel';",
|
||||
"WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Curr.MisNumber = nl.MisNumber AND MisData_Curr.RevID < nl.RevID AND MisData_Curr.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;",
|
||||
"MERGE INTO dbo.MisData_Hist AS target USING (SELECT * FROM dbo.MisData_Curr WHERE Status = 'BackLevel') AS source ON target.ItemNumber = source.ItemNumber AND target.BranchCode = source.BranchCode AND target.SequenceNumber = source.SequenceNumber AND target.MisNumber = source.MisNumber AND target.CharNumber = source.CharNumber WHEN MATCHED THEN UPDATE SET target.RevID = source.RevID, target.TestDescription = source.TestDescription, target.SamplingType = source.SamplingType, target.SamplingValue = source.SamplingValue, target.ToolsGauges = source.ToolsGauges, target.WorkInstructions = source.WorkInstructions, target.Status = source.Status, target.ReleaseDate = source.ReleaseDate, target.ObsoleteDate = source.ObsoleteDate WHEN NOT MATCHED THEN INSERT (ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, Status, ReleaseDate, ObsoleteDate) VALUES (source.ItemNumber, source.BranchCode, source.SequenceNumber, source.MisNumber, source.RevID, source.CharNumber, source.TestDescription, source.SamplingType, source.SamplingValue, source.ToolsGauges, source.WorkInstructions, source.Status, source.ReleaseDate, source.ObsoleteDate);",
|
||||
"DELETE FROM dbo.MisData_Curr WHERE Status = 'BackLevel';",
|
||||
"ALTER INDEX [PK_MisData_Curr] ON [dbo].[MisData_Curr] REBUILD;"
|
||||
]
|
||||
},
|
||||
"MisData_Hist": {
|
||||
"source": {
|
||||
"connection": "cms",
|
||||
"query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel' AND Status.PDATE_RELEASED >= :lastUpdateDT",
|
||||
"massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel'",
|
||||
"parameters": {
|
||||
"lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": { "intervalMinutes": 100800 },
|
||||
"daily": { "enabled": false },
|
||||
"hourly": { "enabled": false }
|
||||
},
|
||||
"destination": {
|
||||
"table": "MisData_Hist",
|
||||
"matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"]
|
||||
},
|
||||
"postScripts": [
|
||||
"SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Hist.MisNumber = bl.MisNumber AND MisData_Hist.RevID = bl.RevID AND MisData_Hist.Status = 'Current' AND bl.Status = 'BackLevel';",
|
||||
"WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Hist.MisNumber = nl.MisNumber AND MisData_Hist.RevID < nl.RevID AND MisData_Hist.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;",
|
||||
"ALTER INDEX [PK_MisData_Hist] ON [dbo].[MisData_Hist] REBUILD;"
|
||||
]
|
||||
},
|
||||
"WorkOrderTime_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot WHERE (wot.DATEUPDATED_WTUPMJ > :dateUpdated OR (wot.DATEUPDATED_WTUPMJ = :dateUpdated AND wot.TIMEOFDAY_WTTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderTime_Curr",
|
||||
"matchColumns": ["UniqueID"],
|
||||
"excludeFromUpdate": ["UniqueID", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"WorkOrderComponent_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL AND (woc.DATEUPDATED_WMUPMJ > :dateUpdated OR (woc.DATEUPDATED_WMUPMJ = :dateUpdated AND woc.TIMEOFDAY_WMTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderComponent_Curr",
|
||||
"matchColumns": ["UniqueID"],
|
||||
"excludeFromUpdate": ["UniqueID", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"WorkOrderStep_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY) WHERE (wos.DATEUPDATED_WLUPMJ > :dateUpdated OR (wos.DATEUPDATED_WLUPMJ = :dateUpdated AND wos.TIMEOFDAY_WLTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY)",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderStep_Curr",
|
||||
"matchColumns": ["WorkOrderNumber", "BranchCode", "StepNumber"],
|
||||
"excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "StepNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"WorkOrderRouting": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410' AND (woz.DATEUPDATED_SZUPMJ > :dateUpdated OR (woz.DATEUPDATED_SZUPMJ = :dateUpdated AND woz.TIMEOFDAY_SZTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderRouting",
|
||||
"matchColumns": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber"],
|
||||
"excludeFromUpdate": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"StatusCode": {
|
||||
"source": {
|
||||
"connection": "giw",
|
||||
"query": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL AND (sc.DATEUPDATED_DRUPMJ > :dateUpdated OR (sc.DATEUPDATED_DRUPMJ = :dateUpdated AND sc.TIMELASTUPDATED_DRUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "StatusCode",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"OrgHierarchy": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL AND (oh.DATEUPDATED_IWUPMJ > :dateUpdated OR (oh.DATEUPDATED_IWUPMJ = :dateUpdated AND oh.TIMEOFDAY_IWTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "OrgHierarchy",
|
||||
"matchColumns": ["WorkCenterCode", "BranchCode"],
|
||||
"excludeFromUpdate": ["WorkCenterCode", "BranchCode", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"RouteMaster": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL AND (route_master.DATEUPDATED_IRUPMJ > :dateUpdated OR (route_master.DATEUPDATED_IRUPMJ = :dateUpdated AND route_master.TIMEOFDAY_IRTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "RouteMaster",
|
||||
"matchColumns": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber"],
|
||||
"excludeFromUpdate": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"FunctionCode": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code",
|
||||
"massQuery": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code",
|
||||
"parameters": {}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": { "prePurge": true, "reIndex": true },
|
||||
"daily": { "prePurge": true, "reIndex": true },
|
||||
"hourly": { "prePurge": true, "reIndex": true }
|
||||
},
|
||||
"destination": {
|
||||
"table": "FunctionCode",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
using System.Text.Json;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Etl.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Scripts;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
using JdeScoping.DataSync.Etl.Transformers;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Builds executable ETL pipelines from pipeline configuration.
|
||||
/// </summary>
|
||||
public class EtlPipelineBuilderService : IEtlPipelineBuilder
|
||||
{
|
||||
private const string DefaultTimezone = "UTC";
|
||||
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly ILogger<EtlPipeline> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="EtlPipelineBuilderService"/> class.
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">Factory for creating database connections.</param>
|
||||
/// <param name="logger">Logger for pipeline execution.</param>
|
||||
public EtlPipelineBuilderService(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
ILogger<EtlPipeline> logger)
|
||||
{
|
||||
_connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public EtlPipeline Build(EtlPipelineConfig config, UpdateTypes updateType, DateTime? minimumDt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
|
||||
// Determine if this is a mass sync
|
||||
var isMassSync = updateType == UpdateTypes.Mass;
|
||||
|
||||
// Use massQuery for Mass updates if available, otherwise use the regular query
|
||||
var useMassQuery = isMassSync && !string.IsNullOrEmpty(config.Source.MassQuery);
|
||||
|
||||
// Create source with parameter substitution
|
||||
var source = CreateSource(config.Source, minimumDt, useMassQuery);
|
||||
|
||||
// Create destination based on update type
|
||||
// Mass = bulk import (TRUNCATE + INSERT), Daily/Hourly = bulk merge (UPSERT)
|
||||
var destination = isMassSync
|
||||
? CreateBulkImportDestination(config.Destination)
|
||||
: CreateBulkMergeDestination(config.Destination);
|
||||
|
||||
// Build the pipeline
|
||||
var builder = new EtlPipelineBuilder()
|
||||
.WithName(config.Name)
|
||||
.WithSource(source)
|
||||
.WithDestination(destination)
|
||||
.WithLogger(_logger);
|
||||
|
||||
// Add transformers if configured
|
||||
foreach (var transform in config.Transforms)
|
||||
{
|
||||
var transformer = CreateTransformer(transform);
|
||||
if (transformer != null)
|
||||
{
|
||||
builder.WithTransformer(transformer);
|
||||
}
|
||||
}
|
||||
|
||||
// Add pre-scripts: config scripts first, then TRUNCATE for mass sync
|
||||
foreach (var script in config.PreScripts)
|
||||
{
|
||||
builder.WithPreScript(CreateScriptRunner(script));
|
||||
}
|
||||
|
||||
if (isMassSync)
|
||||
{
|
||||
var truncateSql = $"TRUNCATE TABLE [{config.Destination.Table}]";
|
||||
builder.WithPreScript(new SqlScriptRunner(_connectionFactory, truncateSql, "PrePurge"));
|
||||
}
|
||||
|
||||
// Add post-scripts: REBUILD INDEX for mass sync first, then config scripts
|
||||
if (isMassSync)
|
||||
{
|
||||
var reindexSql = $"ALTER INDEX ALL ON [{config.Destination.Table}] REBUILD";
|
||||
builder.WithPostScript(new SqlScriptRunner(_connectionFactory, reindexSql, "ReIndex"));
|
||||
}
|
||||
|
||||
foreach (var script in config.PostScripts)
|
||||
{
|
||||
builder.WithPostScript(CreateScriptRunner(script));
|
||||
}
|
||||
|
||||
return builder.Build();
|
||||
}
|
||||
|
||||
private IImportSource CreateSource(SourceElement sourceConfig, DateTime? minDt, bool useMassQuery)
|
||||
{
|
||||
// Use massQuery if specified, otherwise use the default query
|
||||
var query = useMassQuery ? sourceConfig.MassQuery! : sourceConfig.Query;
|
||||
|
||||
var parameters = new Dictionary<string, object>();
|
||||
var converter = new ParameterFormatConverter(DefaultTimezone);
|
||||
|
||||
// Only add parameters when not using massQuery (mass queries typically don't need date parameters)
|
||||
var needsParameters = !useMassQuery;
|
||||
|
||||
if (sourceConfig.Parameters.Count > 0 && minDt.HasValue && needsParameters)
|
||||
{
|
||||
foreach (var (_, paramConfig) in sourceConfig.Parameters)
|
||||
{
|
||||
var paramValue = paramConfig.Source.ToLowerInvariant() switch
|
||||
{
|
||||
"offset" => converter.Convert(minDt.Value, paramConfig.Format),
|
||||
"static" => paramConfig.Value
|
||||
?? throw new InvalidOperationException(
|
||||
$"Static parameter '{paramConfig.Name}' requires a value."),
|
||||
_ => throw new NotSupportedException(
|
||||
$"Parameter source '{paramConfig.Source}' is not supported.")
|
||||
};
|
||||
|
||||
// Use the parameter name exactly as configured (provider-specific)
|
||||
parameters[paramConfig.Name] = paramValue;
|
||||
}
|
||||
}
|
||||
|
||||
return new DbQuerySource(
|
||||
_connectionFactory,
|
||||
sourceConfig.Connection,
|
||||
query,
|
||||
parameters);
|
||||
}
|
||||
|
||||
private IImportDestination CreateBulkImportDestination(DestinationElement destConfig)
|
||||
{
|
||||
return new DbBulkImportDestination(_connectionFactory, destConfig.Table);
|
||||
}
|
||||
|
||||
private IImportDestination CreateBulkMergeDestination(DestinationElement destConfig)
|
||||
{
|
||||
if (destConfig.MatchColumns.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"matchColumns required for incremental sync on table '{destConfig.Table}'.");
|
||||
}
|
||||
|
||||
return new DbBulkMergeDestination(
|
||||
_connectionFactory,
|
||||
destConfig.Table,
|
||||
destConfig.MatchColumns.ToArray(),
|
||||
updateColumns: null,
|
||||
excludeFromUpdate: destConfig.ExcludeFromUpdate.Count > 0
|
||||
? destConfig.ExcludeFromUpdate.ToArray()
|
||||
: null,
|
||||
updateCondition: null);
|
||||
}
|
||||
|
||||
private IScriptRunner CreateScriptRunner(ScriptElement script)
|
||||
{
|
||||
var name = $"Script:{script.Script.Substring(0, Math.Min(30, script.Script.Length))}";
|
||||
return new SqlScriptRunner(_connectionFactory, script.Script, name);
|
||||
}
|
||||
|
||||
private static IDataTransformer? CreateTransformer(TransformElement transform)
|
||||
{
|
||||
var type = transform.TransformType.ToLowerInvariant();
|
||||
|
||||
return type switch
|
||||
{
|
||||
"columndrop" => CreateColumnDropTransformer(transform),
|
||||
"columnrename" => CreateColumnRenameTransformer(transform),
|
||||
"jdedate" => CreateJdeDateTransformer(transform),
|
||||
"regex" => CreateRegexTransformer(transform),
|
||||
_ => null // Skip unknown transformer types
|
||||
};
|
||||
}
|
||||
|
||||
private static IDataTransformer? CreateColumnDropTransformer(TransformElement transform)
|
||||
{
|
||||
if (transform.Config == null)
|
||||
return null;
|
||||
|
||||
var columns = transform.Config.Value.TryGetProperty("columns", out var columnsElement)
|
||||
? columnsElement.EnumerateArray().Select(e => e.GetString()!).ToArray()
|
||||
: null;
|
||||
|
||||
if (columns == null || columns.Length == 0)
|
||||
return null;
|
||||
|
||||
return new ColumnDropTransformer(columns);
|
||||
}
|
||||
|
||||
private static IDataTransformer? CreateColumnRenameTransformer(TransformElement transform)
|
||||
{
|
||||
if (transform.Config == null)
|
||||
return null;
|
||||
|
||||
var renames = new List<(string OldName, string NewName)>();
|
||||
|
||||
if (transform.Config.Value.TryGetProperty("mappings", out var mappingsElement))
|
||||
{
|
||||
foreach (var prop in mappingsElement.EnumerateObject())
|
||||
{
|
||||
renames.Add((prop.Name, prop.Value.GetString() ?? prop.Name));
|
||||
}
|
||||
}
|
||||
|
||||
if (renames.Count == 0)
|
||||
return null;
|
||||
|
||||
return new ColumnRenameTransformer(renames.ToArray());
|
||||
}
|
||||
|
||||
private static IDataTransformer? CreateJdeDateTransformer(TransformElement transform)
|
||||
{
|
||||
if (transform.Config == null)
|
||||
return null;
|
||||
|
||||
var dateColumn = transform.Config.Value.TryGetProperty("dateColumn", out var dateEl)
|
||||
? dateEl.GetString()
|
||||
: null;
|
||||
var timeColumn = transform.Config.Value.TryGetProperty("timeColumn", out var timeEl)
|
||||
? timeEl.GetString()
|
||||
: null;
|
||||
var outputColumn = transform.Config.Value.TryGetProperty("outputColumn", out var outputEl)
|
||||
? outputEl.GetString()
|
||||
: null;
|
||||
|
||||
if (string.IsNullOrEmpty(dateColumn) || string.IsNullOrEmpty(timeColumn) || string.IsNullOrEmpty(outputColumn))
|
||||
return null;
|
||||
|
||||
return new JdeDateTransformer(dateColumn, timeColumn, outputColumn);
|
||||
}
|
||||
|
||||
private static IDataTransformer? CreateRegexTransformer(TransformElement transform)
|
||||
{
|
||||
if (transform.Config == null)
|
||||
return null;
|
||||
|
||||
var column = transform.Config.Value.TryGetProperty("column", out var columnElement)
|
||||
? columnElement.GetString()
|
||||
: null;
|
||||
var pattern = transform.Config.Value.TryGetProperty("pattern", out var patternElement)
|
||||
? patternElement.GetString()
|
||||
: null;
|
||||
var replacement = transform.Config.Value.TryGetProperty("replacement", out var replacementElement)
|
||||
? replacementElement.GetString()
|
||||
: string.Empty;
|
||||
|
||||
if (string.IsNullOrEmpty(column) || string.IsNullOrEmpty(pattern))
|
||||
return null;
|
||||
|
||||
return new RegexTransformer(column, pattern, replacement ?? string.Empty);
|
||||
}
|
||||
}
|
||||
@@ -1,370 +0,0 @@
|
||||
using System.Text.Json;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Scripts;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Factory for creating ETL pipelines from JSON configuration.
|
||||
/// </summary>
|
||||
public class EtlPipelineFactory : IEtlPipelineFactory
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
AllowTrailingCommas = true
|
||||
};
|
||||
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly ILogger<EtlPipeline> _logger;
|
||||
private readonly PipelinesRoot _config;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new pipeline factory.
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">Factory for creating database connections.</param>
|
||||
/// <param name="options">Pipeline configuration options.</param>
|
||||
/// <param name="logger">Logger for pipeline execution.</param>
|
||||
public EtlPipelineFactory(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
IOptions<PipelineOptions> options,
|
||||
ILogger<EtlPipeline> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
_connectionFactory = connectionFactory;
|
||||
_logger = logger;
|
||||
_config = LoadPipelineConfigs(options.Value.ConfigPath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new pipeline factory with a pre-loaded configuration (for testing).
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">Factory for creating database connections.</param>
|
||||
/// <param name="config">Pre-loaded pipeline configuration.</param>
|
||||
/// <param name="logger">Logger for pipeline execution.</param>
|
||||
internal EtlPipelineFactory(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
PipelinesRoot config,
|
||||
ILogger<EtlPipeline> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
ValidateConfig(config);
|
||||
|
||||
_connectionFactory = connectionFactory;
|
||||
_logger = logger;
|
||||
_config = config;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IEtlPipelineBuilder ForTable(string tableName)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tableName);
|
||||
|
||||
if (!_config.Pipelines.TryGetValue(tableName, out var pipelineConfig))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"No pipeline configured for table: {tableName}. " +
|
||||
$"Available tables: {string.Join(", ", _config.Pipelines.Keys)}");
|
||||
}
|
||||
|
||||
return new PipelineBuilder(
|
||||
_connectionFactory,
|
||||
tableName,
|
||||
pipelineConfig,
|
||||
_config.EffectiveSettings,
|
||||
_config.EffectiveScheduleDefaults,
|
||||
_logger);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyList<string> GetAvailableTables()
|
||||
{
|
||||
return _config.Pipelines.Keys.ToList().AsReadOnly();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public PipelineConfig? GetPipelineConfig(string tableName)
|
||||
{
|
||||
return _config.Pipelines.TryGetValue(tableName, out var config) ? config : null;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ScheduleDefaults GetScheduleDefaults()
|
||||
{
|
||||
return _config.EffectiveScheduleDefaults;
|
||||
}
|
||||
|
||||
private PipelinesRoot LoadPipelineConfigs(string configPath)
|
||||
{
|
||||
// Resolve path relative to assembly location (handles both debug and publish)
|
||||
var assemblyDir = Path.GetDirectoryName(typeof(EtlPipelineFactory).Assembly.Location)!;
|
||||
var fullPath = Path.Combine(assemblyDir, configPath);
|
||||
|
||||
if (!File.Exists(fullPath))
|
||||
{
|
||||
throw new FileNotFoundException(
|
||||
$"Pipeline config not found: {fullPath}. " +
|
||||
"Ensure the config file is included in the build output.");
|
||||
}
|
||||
|
||||
var json = File.ReadAllText(fullPath);
|
||||
var root = JsonSerializer.Deserialize<PipelinesRoot>(json, JsonOptions)
|
||||
?? throw new InvalidOperationException("Failed to deserialize pipeline config: result was null.");
|
||||
|
||||
ValidateConfig(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
private static void ValidateConfig(PipelinesRoot root)
|
||||
{
|
||||
foreach (var (name, config) in root.Pipelines)
|
||||
{
|
||||
// Schedules are now required
|
||||
if (config.Schedules == null)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Pipeline '{name}' must define 'schedules'.");
|
||||
}
|
||||
|
||||
// Validate no runtime parameters (not yet supported)
|
||||
if (config.Source.Parameters != null)
|
||||
{
|
||||
foreach (var (paramName, paramConfig) in config.Source.Parameters)
|
||||
{
|
||||
if (paramConfig.Source.Equals("runtime", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new NotSupportedException(
|
||||
$"Pipeline '{name}' parameter '{paramName}': " +
|
||||
"runtime parameter source is not yet supported.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PipelineBuilder : IEtlPipelineBuilder
|
||||
{
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly string _tableName;
|
||||
private readonly PipelineConfig _config;
|
||||
private readonly PipelineSettings _settings;
|
||||
private readonly ScheduleDefaults _scheduleDefaults;
|
||||
private readonly ILogger<EtlPipeline> _logger;
|
||||
private UpdateTypes _updateType = UpdateTypes.Hourly;
|
||||
private DateTime? _minDtOverride;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the PipelineBuilder class.
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">Factory for creating database connections.</param>
|
||||
/// <param name="tableName">The name of the table for this pipeline.</param>
|
||||
/// <param name="config">The pipeline configuration.</param>
|
||||
/// <param name="settings">Global pipeline settings.</param>
|
||||
/// <param name="scheduleDefaults">Default schedule configuration.</param>
|
||||
/// <param name="logger">Logger for pipeline execution.</param>
|
||||
public PipelineBuilder(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
string tableName,
|
||||
PipelineConfig config,
|
||||
PipelineSettings settings,
|
||||
ScheduleDefaults scheduleDefaults,
|
||||
ILogger<EtlPipeline> logger)
|
||||
{
|
||||
_connectionFactory = connectionFactory;
|
||||
_tableName = tableName;
|
||||
_config = config;
|
||||
_settings = settings;
|
||||
_scheduleDefaults = scheduleDefaults;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Specifies the update type for this pipeline.
|
||||
/// </summary>
|
||||
/// <param name="updateType">The type of update (Mass, Daily, or Hourly).</param>
|
||||
/// <returns>The builder for fluent configuration.</returns>
|
||||
public IEtlPipelineBuilder WithUpdateType(UpdateTypes updateType)
|
||||
{
|
||||
_updateType = updateType;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Specifies the minimum date for incremental data extraction.
|
||||
/// </summary>
|
||||
/// <param name="minDt">The minimum date, or null for no filter.</param>
|
||||
/// <returns>The builder for fluent configuration.</returns>
|
||||
public IEtlPipelineBuilder WithMinimumDate(DateTime? minDt)
|
||||
{
|
||||
_minDtOverride = minDt;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds and returns the configured ETL pipeline.
|
||||
/// </summary>
|
||||
/// <returns>A configured ETL pipeline ready for execution.</returns>
|
||||
public EtlPipeline Build()
|
||||
{
|
||||
return BuildWithSchedules();
|
||||
}
|
||||
|
||||
private EtlPipeline BuildWithSchedules()
|
||||
{
|
||||
var scheduleConfig = GetEffectiveScheduleConfig(_updateType);
|
||||
|
||||
// Compute MinDt from override
|
||||
var minDt = _minDtOverride;
|
||||
|
||||
// Use massQuery for Mass, regular query for Daily/Hourly
|
||||
var useMassQuery = _updateType == UpdateTypes.Mass && !string.IsNullOrEmpty(_config.Source.MassQuery);
|
||||
|
||||
// Create source with parameter substitution
|
||||
var source = CreateSource(_config.Source, minDt, useMassQuery);
|
||||
|
||||
// Determine destination type (Mass with prePurge = bulkImport, others = bulkMerge unless prePurge)
|
||||
var destType = scheduleConfig.PrePurge ? "bulkImport" : "bulkMerge";
|
||||
var destination = CreateDestination(destType, _config.Destination, scheduleConfig);
|
||||
|
||||
// Build pipeline with scripts
|
||||
var builder = new EtlPipelineBuilder()
|
||||
.WithName(_tableName)
|
||||
.WithSource(source)
|
||||
.WithDestination(destination)
|
||||
.WithLogger(_logger);
|
||||
|
||||
// Add pre-scripts: config scripts first, then prePurge
|
||||
foreach (var script in _config.PreScripts ?? [])
|
||||
{
|
||||
builder.WithPreScript(new SqlScriptRunner(_connectionFactory, script, $"PreScript:{script.Substring(0, Math.Min(30, script.Length))}"));
|
||||
}
|
||||
|
||||
if (scheduleConfig.PrePurge)
|
||||
{
|
||||
var truncateSql = $"TRUNCATE TABLE [{_config.Destination.Table}]";
|
||||
builder.WithPreScript(new SqlScriptRunner(_connectionFactory, truncateSql, "PrePurge"));
|
||||
}
|
||||
|
||||
// Add post-scripts: reIndex first, then config scripts
|
||||
if (scheduleConfig.ReIndex)
|
||||
{
|
||||
var reindexSql = $"ALTER INDEX ALL ON [{_config.Destination.Table}] REBUILD";
|
||||
builder.WithPostScript(new SqlScriptRunner(_connectionFactory, reindexSql, "ReIndex"));
|
||||
}
|
||||
|
||||
foreach (var script in _config.PostScripts ?? [])
|
||||
{
|
||||
builder.WithPostScript(new SqlScriptRunner(_connectionFactory, script, $"PostScript:{script.Substring(0, Math.Min(30, script.Length))}"));
|
||||
}
|
||||
|
||||
return builder.Build();
|
||||
}
|
||||
|
||||
private Configuration.ScheduleConfig GetEffectiveScheduleConfig(UpdateTypes updateType)
|
||||
{
|
||||
// Get default for this update type
|
||||
var defaultConfig = updateType switch
|
||||
{
|
||||
UpdateTypes.Mass => _scheduleDefaults.Mass,
|
||||
UpdateTypes.Daily => _scheduleDefaults.Daily,
|
||||
UpdateTypes.Hourly => _scheduleDefaults.Hourly,
|
||||
_ => _scheduleDefaults.Hourly
|
||||
};
|
||||
|
||||
// Get pipeline-specific override if exists
|
||||
var pipelineConfig = updateType switch
|
||||
{
|
||||
UpdateTypes.Mass => _config.Schedules?.Mass,
|
||||
UpdateTypes.Daily => _config.Schedules?.Daily,
|
||||
UpdateTypes.Hourly => _config.Schedules?.Hourly,
|
||||
_ => null
|
||||
};
|
||||
|
||||
// Merge: pipeline config overrides defaults
|
||||
return pipelineConfig?.MergeWith(defaultConfig) ?? defaultConfig;
|
||||
}
|
||||
|
||||
private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt, bool useMassQuery)
|
||||
{
|
||||
// Use massQuery if specified, otherwise use the default query
|
||||
var query = useMassQuery ? sourceConfig.MassQuery! : sourceConfig.Query;
|
||||
|
||||
var parameters = new Dictionary<string, object>();
|
||||
var converter = new ParameterFormatConverter(_settings.Timezone);
|
||||
|
||||
// Only add parameters when not using massQuery (mass queries typically don't need date parameters)
|
||||
var needsParameters = !useMassQuery;
|
||||
|
||||
if (sourceConfig.Parameters != null && minDt.HasValue && needsParameters)
|
||||
{
|
||||
foreach (var (_, paramConfig) in sourceConfig.Parameters)
|
||||
{
|
||||
var paramValue = paramConfig.Source.ToLowerInvariant() switch
|
||||
{
|
||||
"offset" => converter.Convert(minDt.Value, paramConfig.Format),
|
||||
"static" => paramConfig.Value
|
||||
?? throw new InvalidOperationException(
|
||||
$"Static parameter '{paramConfig.Name}' requires a value."),
|
||||
_ => throw new NotSupportedException(
|
||||
$"Parameter source '{paramConfig.Source}' is not supported.")
|
||||
};
|
||||
|
||||
// Use the parameter name exactly as configured (provider-specific)
|
||||
parameters[paramConfig.Name] = paramValue;
|
||||
}
|
||||
}
|
||||
|
||||
return new DbQuerySource(
|
||||
_connectionFactory,
|
||||
sourceConfig.Connection,
|
||||
query,
|
||||
parameters);
|
||||
}
|
||||
|
||||
private IImportDestination CreateDestination(
|
||||
string destType,
|
||||
DestinationConfig baseConfig,
|
||||
Configuration.ScheduleConfig scheduleConfig)
|
||||
{
|
||||
var tableName = baseConfig.Table;
|
||||
|
||||
// Use base config for match/exclude columns
|
||||
var matchColumns = baseConfig.MatchColumns?.ToArray();
|
||||
var excludeFromUpdate = baseConfig.ExcludeFromUpdate?.ToArray();
|
||||
|
||||
return destType.ToLowerInvariant() switch
|
||||
{
|
||||
"bulkimport" => new DbBulkImportDestination(_connectionFactory, tableName),
|
||||
|
||||
"bulkmerge" => new DbBulkMergeDestination(
|
||||
_connectionFactory,
|
||||
tableName,
|
||||
matchColumns ?? throw new InvalidOperationException(
|
||||
$"matchColumns required for bulkMerge destination on table '{tableName}'."),
|
||||
updateColumns: null,
|
||||
excludeFromUpdate: excludeFromUpdate,
|
||||
updateCondition: scheduleConfig.UpdateWhen),
|
||||
|
||||
_ => throw new InvalidOperationException(
|
||||
$"Unknown destination type: '{destType}'. Expected 'bulkImport' or 'bulkMerge'.")
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
|
||||
namespace JdeScoping.DataSync.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Builds executable ETL pipelines from pipeline configuration.
|
||||
/// </summary>
|
||||
public interface IEtlPipelineBuilder
|
||||
{
|
||||
/// <summary>
|
||||
/// Builds an executable ETL pipeline from the given configuration.
|
||||
/// </summary>
|
||||
/// <param name="config">The pipeline configuration.</param>
|
||||
/// <param name="updateType">The update type (Mass, Daily, or Hourly).</param>
|
||||
/// <param name="minimumDt">The minimum date for incremental syncs, or null for mass syncs.</param>
|
||||
/// <returns>A configured ETL pipeline ready for execution.</returns>
|
||||
EtlPipeline Build(EtlPipelineConfig config, UpdateTypes updateType, DateTime? minimumDt);
|
||||
}
|
||||
@@ -1,8 +1,7 @@
|
||||
using System.Diagnostics;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.Core.Interfaces;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Models;
|
||||
using JdeScoping.DataSync.Telemetry;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -15,7 +14,7 @@ namespace JdeScoping.DataSync.Services;
|
||||
/// </summary>
|
||||
public class TableSyncOperation : ITableSyncOperation
|
||||
{
|
||||
private readonly IEtlPipelineFactory _pipelineFactory;
|
||||
private readonly IEtlPipelineBuilder _pipelineBuilder;
|
||||
private readonly IDataUpdateRepository _updateRepository;
|
||||
private readonly IOptions<DataSyncOptions> _options;
|
||||
private readonly ILogger<TableSyncOperation> _logger;
|
||||
@@ -24,19 +23,19 @@ public class TableSyncOperation : ITableSyncOperation
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="TableSyncOperation"/> class.
|
||||
/// </summary>
|
||||
/// <param name="pipelineFactory">Factory for creating ETL pipelines.</param>
|
||||
/// <param name="pipelineBuilder">Builder for creating ETL pipelines.</param>
|
||||
/// <param name="updateRepository">Repository for managing data update records.</param>
|
||||
/// <param name="options">Data sync configuration options.</param>
|
||||
/// <param name="logger">Logger for operation events.</param>
|
||||
/// <param name="metrics">Metrics collector for operation tracking.</param>
|
||||
public TableSyncOperation(
|
||||
IEtlPipelineFactory pipelineFactory,
|
||||
IEtlPipelineBuilder pipelineBuilder,
|
||||
IDataUpdateRepository updateRepository,
|
||||
IOptions<DataSyncOptions> options,
|
||||
ILogger<TableSyncOperation> logger,
|
||||
DataSyncMetrics metrics)
|
||||
{
|
||||
_pipelineFactory = pipelineFactory ?? throw new ArgumentNullException(nameof(pipelineFactory));
|
||||
_pipelineBuilder = pipelineBuilder ?? throw new ArgumentNullException(nameof(pipelineBuilder));
|
||||
_updateRepository = updateRepository ?? throw new ArgumentNullException(nameof(updateRepository));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
@@ -162,12 +161,16 @@ public class TableSyncOperation : ITableSyncOperation
|
||||
{
|
||||
_logger.LogDebug("Building pipeline for {Table} with UpdateType={UpdateType}", task.TableName, task.UpdateType);
|
||||
|
||||
// Build and execute the pipeline using the task's UpdateType directly
|
||||
var pipeline = _pipelineFactory
|
||||
.ForTable(task.TableName)
|
||||
.WithUpdateType(task.UpdateType)
|
||||
.WithMinimumDate(task.MinimumDt)
|
||||
.Build();
|
||||
// Ensure the task has a pipeline configuration
|
||||
if (task.Pipeline == null)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"No pipeline configuration for {task.TableName}. " +
|
||||
"Ensure the task was created with a valid EtlPipelineConfig.");
|
||||
}
|
||||
|
||||
// Build and execute the pipeline using the task's pipeline configuration
|
||||
var pipeline = _pipelineBuilder.Build(task.Pipeline, task.UpdateType, task.MinimumDt);
|
||||
|
||||
var result = await pipeline.ExecuteAsync(cancellationToken);
|
||||
|
||||
|
||||
@@ -1,392 +0,0 @@
|
||||
{
|
||||
"settings": {
|
||||
"timezone": "UTC"
|
||||
},
|
||||
"scheduleDefaults": {
|
||||
"mass": { "enabled": true, "intervalMinutes": 10080, "prePurge": true, "reIndex": true },
|
||||
"daily": { "enabled": true, "intervalMinutes": 1440, "prePurge": false, "reIndex": false },
|
||||
"hourly": { "enabled": true, "intervalMinutes": 60, "prePurge": false, "reIndex": false }
|
||||
},
|
||||
"pipelines": {
|
||||
"WorkOrder_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo WHERE (wo.WAUPMJ > :dateUpdated OR (wo.WAUPMJ = :dateUpdated AND wo.WATDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT wo.WADOCO AS WorkOrderNumber, TRIM(wo.WAMMCU) AS BranchCode, TRIM(wo.WALOTN) AS LotNumber, TRIM(wo.WALITM) AS ItemNumber, wo.WAITM AS ShortItemNumber, TRIM(wo.WAPARS) AS ParentWorkOrderNumber, wo.WAUORG / 100.0 AS OrderQuantity, wo.WASOBK / 100.0 AS HeldQuantity, wo.WASOQS / 100.0 AS ShippedQuantity, TRIM(wo.WASRST) AS StatusCode, CASE wo.WADCG WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WADCG+1900000,'YYYYDDD') END AS StatusCodeUpdateDT, CASE wo.WATRDJ WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WATRDJ+1900000,'YYYYDDD') END AS IssueDate, CASE wo.WASTRT WHEN 0 THEN TO_DATE('1900-01-01', 'YYYY-MM-DD') ELSE TO_DATE(wo.WASTRT+1900000,'YYYYDDD') END AS StartDate, TRIM(wo.WATRT) AS RoutingType, wo.WAUPMJ AS LastUpdateDate, wo.WATDAY AS LastUpdateTime FROM {ProductionSchema}.F4801 wo",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrder_Curr",
|
||||
"matchColumns": ["WorkOrderNumber", "BranchCode"],
|
||||
"excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"Lot": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL AND (lot.IOUPMJ > :dateUpdated OR (lot.IOUPMJ = :dateUpdated AND lot.IOTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(lot.IOLOTN) AS LotNumber, TRIM(lot.IOMCU) AS BranchCode, lot.IOITM AS ShortItemNumber, TRIM(lot.IOLITM) AS ItemNumber, lot.IOVEND AS SupplierCode, lot.IOLOTS AS StatusCode, TRIM(lot.IOLOT1) AS Memo1, TRIM(lot.IOLOT2) AS Memo2, TRIM(lot.IOLOT3) AS Memo3, lot.IOUPMJ AS LastUpdateDate, lot.IOTDAY AS LastUpdateTime FROM {ProductionSchema}.F4108 lot WHERE TRIM(lot.IOLOTN) IS NOT NULL AND TRIM(lot.IOMCU) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "Lot",
|
||||
"matchColumns": ["LotNumber", "BranchCode"],
|
||||
"excludeFromUpdate": ["LotNumber", "BranchCode", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"LotUsage_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL AND (lu.ILTRDJ > :dateUpdated OR (lu.ILTRDJ = :dateUpdated AND lu.ILTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT lu.ILUKID AS UniqueId, lu.ILDOCO AS WorkOrderNumber, TRIM(lu.ILLOTN) AS LotNumber, TRIM(lu.ILMCU) AS BranchCode, lu.ILITM AS ShortItemNumber, lu.ILTRQT AS Quantity, lu.ILTRDJ AS LastUpdateDate, lu.ILTDAY AS LastUpdateTime FROM {ProductionSchema}.F4111 lu WHERE lu.ILDCT = 'IM' AND TRIM(lu.ILLOTN) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "LotUsage_Curr",
|
||||
"matchColumns": ["UniqueId"],
|
||||
"excludeFromUpdate": ["UniqueId", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"Item": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL AND (pn.IMUPMJ > :dateUpdated OR (pn.IMUPMJ = :dateUpdated AND pn.IMTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT pn.IMITM AS ShortItemNumber, TRIM(pn.IMLITM) AS ItemNumber, TRIM(pn.IMDSC1) AS Description, TRIM(pn.IMPRP4) AS PlanningFamily, TRIM(pn.IMSTKT) AS StockingType, pn.IMUPMJ AS LastUpdateDate, pn.IMTDAY AS LastUpdateTime FROM {ProductionSchema}.F4101 pn WHERE TRIM(pn.IMLITM) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "Item",
|
||||
"matchColumns": ["ShortItemNumber"],
|
||||
"excludeFromUpdate": ["ShortItemNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"WorkCenter": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'WC'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkCenter",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"ProfitCenter": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'I3'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "ProfitCenter",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"JdeUser": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1",
|
||||
"massQuery": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1",
|
||||
"parameters": {}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "JdeUser",
|
||||
"matchColumns": ["AddressNumber"],
|
||||
"excludeFromUpdate": ["AddressNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"Branch": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP' AND (wc.MCUPMJ > :dateUpdated OR (wc.MCUPMJ = :dateUpdated AND wc.MCUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(wc.MCMCU) AS Code, TRIM(wc.MCDL01) AS Description, wc.MCUPMJ AS LastUpdateDate, wc.MCUPMT AS LastUpdateTime FROM {ProductionSchema}.F0006 wc WHERE wc.MCSTYL = 'BP'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "Branch",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"MisData_Curr": {
|
||||
"source": {
|
||||
"connection": "cms",
|
||||
"query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current' AND Status.PDATE_RELEASED >= :lastUpdateDT",
|
||||
"massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current'",
|
||||
"parameters": {
|
||||
"lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": { "intervalMinutes": 100800 },
|
||||
"daily": {},
|
||||
"hourly": { "enabled": false }
|
||||
},
|
||||
"destination": {
|
||||
"table": "MisData_Curr",
|
||||
"matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"]
|
||||
},
|
||||
"postScripts": [
|
||||
"SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Curr.MisNumber = bl.MisNumber AND MisData_Curr.RevID = bl.RevID AND MisData_Curr.Status = 'Current' AND bl.Status = 'BackLevel';",
|
||||
"WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Curr.MisNumber = nl.MisNumber AND MisData_Curr.RevID < nl.RevID AND MisData_Curr.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;",
|
||||
"MERGE INTO dbo.MisData_Hist AS target USING (SELECT * FROM dbo.MisData_Curr WHERE Status = 'BackLevel') AS source ON target.ItemNumber = source.ItemNumber AND target.BranchCode = source.BranchCode AND target.SequenceNumber = source.SequenceNumber AND target.MisNumber = source.MisNumber AND target.CharNumber = source.CharNumber WHEN MATCHED THEN UPDATE SET target.RevID = source.RevID, target.TestDescription = source.TestDescription, target.SamplingType = source.SamplingType, target.SamplingValue = source.SamplingValue, target.ToolsGauges = source.ToolsGauges, target.WorkInstructions = source.WorkInstructions, target.Status = source.Status, target.ReleaseDate = source.ReleaseDate, target.ObsoleteDate = source.ObsoleteDate WHEN NOT MATCHED THEN INSERT (ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, Status, ReleaseDate, ObsoleteDate) VALUES (source.ItemNumber, source.BranchCode, source.SequenceNumber, source.MisNumber, source.RevID, source.CharNumber, source.TestDescription, source.SamplingType, source.SamplingValue, source.ToolsGauges, source.WorkInstructions, source.Status, source.ReleaseDate, source.ObsoleteDate);",
|
||||
"DELETE FROM dbo.MisData_Curr WHERE Status = 'BackLevel';",
|
||||
"ALTER INDEX [PK_MisData_Curr] ON [dbo].[MisData_Curr] REBUILD;"
|
||||
]
|
||||
},
|
||||
"MisData_Hist": {
|
||||
"source": {
|
||||
"connection": "cms",
|
||||
"query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel' AND Status.PDATE_RELEASED >= :lastUpdateDT",
|
||||
"massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel'",
|
||||
"parameters": {
|
||||
"lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": { "intervalMinutes": 100800 },
|
||||
"daily": { "enabled": false },
|
||||
"hourly": { "enabled": false }
|
||||
},
|
||||
"destination": {
|
||||
"table": "MisData_Hist",
|
||||
"matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"]
|
||||
},
|
||||
"postScripts": [
|
||||
"SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Hist.MisNumber = bl.MisNumber AND MisData_Hist.RevID = bl.RevID AND MisData_Hist.Status = 'Current' AND bl.Status = 'BackLevel';",
|
||||
"WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Hist.MisNumber = nl.MisNumber AND MisData_Hist.RevID < nl.RevID AND MisData_Hist.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;",
|
||||
"ALTER INDEX [PK_MisData_Hist] ON [dbo].[MisData_Hist] REBUILD;"
|
||||
]
|
||||
},
|
||||
"WorkOrderTime_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot WHERE (wot.DATEUPDATED_WTUPMJ > :dateUpdated OR (wot.DATEUPDATED_WTUPMJ = :dateUpdated AND wot.TIMEOFDAY_WTTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderTime_Curr",
|
||||
"matchColumns": ["UniqueID"],
|
||||
"excludeFromUpdate": ["UniqueID", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"WorkOrderComponent_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL AND (woc.DATEUPDATED_WMUPMJ > :dateUpdated OR (woc.DATEUPDATED_WMUPMJ = :dateUpdated AND woc.TIMEOFDAY_WMTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, TRIM(woc.LOT_WMLOTN) AS LotNumber, TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, woc.DATEUPDATED_WMUPMJ AS DateUpdated, woc.TIMEOFDAY_WMTDAY AS TimeUpdated FROM JDESTAGE.F3111_VIEW woc WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderComponent_Curr",
|
||||
"matchColumns": ["UniqueID"],
|
||||
"excludeFromUpdate": ["UniqueID", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"WorkOrderStep_Curr": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY) WHERE (wos.DATEUPDATED_WLUPMJ > :dateUpdated OR (wos.DATEUPDATED_WLUPMJ = :dateUpdated AND wos.TIMEOFDAY_WLTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, wos.DATEUPDATED_WLUPMJ AS DateUpdated, wos.TIMEOFDAY_WLTDAY AS TimeUpdated FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY)",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderStep_Curr",
|
||||
"matchColumns": ["WorkOrderNumber", "BranchCode", "StepNumber"],
|
||||
"excludeFromUpdate": ["WorkOrderNumber", "BranchCode", "StepNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"WorkOrderRouting": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410' AND (woz.DATEUPDATED_SZUPMJ > :dateUpdated OR (woz.DATEUPDATED_SZUPMJ = :dateUpdated AND woz.TIMEOFDAY_SZTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, woz.EDILINENUMBER_SZEDLN AS LineNumber, woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, woz.DATEUPDATED_SZUPMJ AS DateUpdated, woz.TIMEOFDAY_SZTDAY AS TimeUpdated FROM JDESTAGE.F3112Z1_VIEW woz WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND woz.DIRECTIONINDICATOR_SZDRIN = '2' AND woz.TRANSACTIONACTION_SZTNAC = '02' AND woz.PROGRAMID_SZPID = 'ER31410'",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "WorkOrderRouting",
|
||||
"matchColumns": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber"],
|
||||
"excludeFromUpdate": ["UserID", "BatchNumber", "TransactionNumber", "LineNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"StatusCode": {
|
||||
"source": {
|
||||
"connection": "giw",
|
||||
"query": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL AND (sc.DATEUPDATED_DRUPMJ > :dateUpdated OR (sc.DATEUPDATED_DRUPMJ = :dateUpdated AND sc.TIMELASTUPDATED_DRUPMT >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS Code, TRIM(sc.DESCRIPTION001_DRDL01) AS Description, sc.DATEUPDATED_DRUPMJ AS DateUpdated, sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated FROM JDESTAGE.F0005_VIEW sc WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND sc.USERDEFINEDCODES_DRRT = 'SS' AND TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "StatusCode",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"OrgHierarchy": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL AND (oh.DATEUPDATED_IWUPMJ > :dateUpdated OR (oh.DATEUPDATED_IWUPMJ = :dateUpdated AND oh.TIMEOFDAY_IWTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, oh.DATEUPDATED_IWUPMJ AS DateUpdated, oh.TIMEOFDAY_IWTDAY AS TimeUpdated FROM JDESTAGE.F30006_VIEW oh WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "OrgHierarchy",
|
||||
"matchColumns": ["WorkCenterCode", "BranchCode"],
|
||||
"excludeFromUpdate": ["WorkCenterCode", "BranchCode", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"RouteMaster": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL AND (route_master.DATEUPDATED_IRUPMJ > :dateUpdated OR (route_master.DATEUPDATED_IRUPMJ = :dateUpdated AND route_master.TIMEOFDAY_IRTDAY >= :timeUpdated))",
|
||||
"massQuery": "SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, route_master.DATEUPDATED_IRUPMJ AS DateUpdated, route_master.TIMEOFDAY_IRTDAY AS TimeUpdated FROM JDESTAGE.F3003_VIEW route_master WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL",
|
||||
"parameters": {
|
||||
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
|
||||
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
|
||||
}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": {},
|
||||
"daily": {},
|
||||
"hourly": {}
|
||||
},
|
||||
"destination": {
|
||||
"table": "RouteMaster",
|
||||
"matchColumns": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber"],
|
||||
"excludeFromUpdate": ["BranchCode", "ItemNumber", "RoutingType", "SequenceNumber", "LastUpdateDt"]
|
||||
}
|
||||
},
|
||||
"FunctionCode": {
|
||||
"source": {
|
||||
"connection": "jde",
|
||||
"query": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code",
|
||||
"massQuery": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code",
|
||||
"parameters": {}
|
||||
},
|
||||
"schedules": {
|
||||
"mass": { "prePurge": true, "reIndex": true },
|
||||
"daily": { "prePurge": true, "reIndex": true },
|
||||
"hourly": { "prePurge": true, "reIndex": true }
|
||||
},
|
||||
"destination": {
|
||||
"table": "FunctionCode",
|
||||
"matchColumns": ["Code"],
|
||||
"excludeFromUpdate": ["Code", "LastUpdateDt"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,9 +63,6 @@
|
||||
"CmsPassword"
|
||||
]
|
||||
},
|
||||
"Pipelines": {
|
||||
"ConfigPath": "Pipelines/pipelines.json"
|
||||
},
|
||||
"WorkProcessor": {
|
||||
"Enabled": true,
|
||||
"WorkInterval": "00:00:05",
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<PackageReference Include="Avalonia.Fonts.Inter" Version="11.2.*" />
|
||||
<PackageReference Include="Avalonia.Diagnostics" Version="11.2.*" Condition="'$(Configuration)' == 'Debug'" />
|
||||
<PackageReference Include="DiffPlex" Version="1.7.*" />
|
||||
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.0.*" />
|
||||
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.1.*" />
|
||||
<PackageReference Include="MessageBox.Avalonia" Version="3.1.*" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.*" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.*" />
|
||||
@@ -31,6 +31,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\JdeScoping.Core\JdeScoping.Core.csproj" />
|
||||
<ProjectReference Include="..\..\JdeScoping.DataSync\JdeScoping.DataSync.csproj" />
|
||||
<ProjectReference Include="..\..\JdeScoping.Infrastructure\JdeScoping.Infrastructure.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -273,7 +273,7 @@ public class SecureStoreSection
|
||||
public class PipelinesSection
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the path to the pipelines configuration file.
|
||||
/// Gets or sets the directory containing pipeline.*.json files.
|
||||
/// </summary>
|
||||
public string ConfigPath { get; set; } = "Pipelines/pipelines.json";
|
||||
public string ConfigDirectory { get; set; } = "Pipelines";
|
||||
}
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Root model for pipelines.json configuration.
|
||||
/// </summary>
|
||||
public class PipelinesConfigModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the pipeline settings.
|
||||
/// </summary>
|
||||
public PipelineSettings Settings { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the default schedules for all pipelines.
|
||||
/// </summary>
|
||||
public ScheduleDefaults ScheduleDefaults { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the collection of named pipelines.
|
||||
/// </summary>
|
||||
public Dictionary<string, PipelineModel> Pipelines { get; set; } = new();
|
||||
}
|
||||
|
||||
public class PipelineSettings
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the timezone for scheduling operations.
|
||||
/// </summary>
|
||||
public string Timezone { get; set; } = "UTC";
|
||||
}
|
||||
|
||||
public class ScheduleDefaults
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the default mass data refresh schedule.
|
||||
/// </summary>
|
||||
public ScheduleModel Mass { get; set; } = new() { Enabled = true, IntervalMinutes = 10080, PrePurge = true, ReIndex = true };
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the default daily data refresh schedule.
|
||||
/// </summary>
|
||||
public ScheduleModel Daily { get; set; } = new() { Enabled = true, IntervalMinutes = 1440 };
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the default hourly data refresh schedule.
|
||||
/// </summary>
|
||||
public ScheduleModel Hourly { get; set; } = new() { Enabled = true, IntervalMinutes = 60 };
|
||||
}
|
||||
|
||||
public class PipelineModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the source configuration for data extraction.
|
||||
/// </summary>
|
||||
public PipelineSource Source { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the schedule configurations for this pipeline.
|
||||
/// </summary>
|
||||
public PipelineSchedules Schedules { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets optional data transformers applied between source and destination.
|
||||
/// </summary>
|
||||
public List<TransformerModel>? Transformers { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the destination configuration for data loading.
|
||||
/// </summary>
|
||||
public PipelineDestination Destination { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets optional scripts to execute before pipeline starts.
|
||||
/// </summary>
|
||||
public string[]? PreScripts { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets optional scripts to execute after pipeline completion.
|
||||
/// </summary>
|
||||
public string[]? PostScripts { get; set; }
|
||||
}
|
||||
|
||||
public class PipelineSource
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the source database connection name.
|
||||
/// Used for database sources.
|
||||
/// </summary>
|
||||
public string Connection { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the query to extract data from the source.
|
||||
/// Used for database sources.
|
||||
/// </summary>
|
||||
public string Query { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the optional mass query for full data extraction.
|
||||
/// Used for database sources.
|
||||
/// </summary>
|
||||
public string? MassQuery { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the query parameters and their definitions.
|
||||
/// Used for database sources.
|
||||
/// </summary>
|
||||
public Dictionary<string, ParameterDefinition> Parameters { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the file name for file-based sources.
|
||||
/// Used for Protobuf+Zstd files.
|
||||
/// </summary>
|
||||
public string? FileName { get; set; }
|
||||
}
|
||||
|
||||
public class ParameterDefinition
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the parameter name.
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the optional parameter format string.
|
||||
/// </summary>
|
||||
public string? Format { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the optional parameter source or derivation logic.
|
||||
/// </summary>
|
||||
public string? Source { get; set; }
|
||||
}
|
||||
|
||||
public class PipelineSchedules
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the mass refresh schedule for this pipeline.
|
||||
/// </summary>
|
||||
public ScheduleModel? Mass { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the daily refresh schedule for this pipeline.
|
||||
/// </summary>
|
||||
public ScheduleModel? Daily { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the hourly refresh schedule for this pipeline.
|
||||
/// </summary>
|
||||
public ScheduleModel? Hourly { get; set; }
|
||||
}
|
||||
|
||||
public class PipelineDestination
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the destination type (BulkImport or BulkMerge).
|
||||
/// BulkImport truncates and loads; BulkMerge matches and updates.
|
||||
/// </summary>
|
||||
public string Type { get; set; } = "BulkMerge";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the destination table name.
|
||||
/// </summary>
|
||||
public string Table { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the columns used to match existing records for updates.
|
||||
/// Only used for BulkMerge destination type.
|
||||
/// </summary>
|
||||
public string[] MatchColumns { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the columns to exclude from update operations.
|
||||
/// Only used for BulkMerge destination type.
|
||||
/// </summary>
|
||||
public string[] ExcludeFromUpdate { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a data transformer applied between source and destination.
|
||||
/// </summary>
|
||||
public class TransformerModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the transformer type.
|
||||
/// Supported types: ColumnDrop, ColumnRename, JdeDate.
|
||||
/// </summary>
|
||||
public string Type { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the columns affected by this transformer.
|
||||
/// Used by ColumnDrop (columns to remove) and JdeDate (date/time columns).
|
||||
/// </summary>
|
||||
public List<string>? Columns { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the column mappings for rename operations.
|
||||
/// Used by ColumnRename: OldName → NewName.
|
||||
/// </summary>
|
||||
public Dictionary<string, string>? Mappings { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the date column name for JdeDate transformer.
|
||||
/// </summary>
|
||||
public string? DateColumn { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the time column name for JdeDate transformer.
|
||||
/// </summary>
|
||||
public string? TimeColumn { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the output column name for JdeDate transformer.
|
||||
/// </summary>
|
||||
public string? OutputColumn { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the column name for Regex transformer.
|
||||
/// </summary>
|
||||
public string? ColumnName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the regex pattern for Regex transformer.
|
||||
/// </summary>
|
||||
public string? Pattern { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the replacement string for Regex transformer (null = Match & Extract mode).
|
||||
/// </summary>
|
||||
public string? Replacement { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets whether regex matching is case-insensitive.
|
||||
/// </summary>
|
||||
public bool IgnoreCase { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the behavior when regex pattern does not match.
|
||||
/// </summary>
|
||||
public NonMatchBehavior NonMatchBehavior { get; set; } = NonMatchBehavior.KeepOriginal;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Specifies behavior when a regex pattern does not match the input value.
|
||||
/// </summary>
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum NonMatchBehavior
|
||||
{
|
||||
/// <summary>Keep the original value unchanged.</summary>
|
||||
KeepOriginal,
|
||||
/// <summary>Return null/DBNull.</summary>
|
||||
ReturnNull,
|
||||
/// <summary>Return an empty string.</summary>
|
||||
ReturnEmpty
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
namespace JdeScoping.ConfigManager.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Model for schedule configuration.
|
||||
/// </summary>
|
||||
public class ScheduleModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the scheduled task is enabled.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the interval in minutes between scheduled task executions.
|
||||
/// </summary>
|
||||
public int IntervalMinutes { get; set; } = 60;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether to purge data before task execution.
|
||||
/// </summary>
|
||||
public bool PrePurge { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether to reindex after task execution.
|
||||
/// </summary>
|
||||
public bool ReIndex { get; set; } = false;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Services;
|
||||
@@ -55,29 +56,6 @@ public class ConfigFileService : IConfigFileService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads the pipelines configuration from the specified file path.
|
||||
/// </summary>
|
||||
/// <param name="path">The file path to load pipelines from.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
/// <returns>The loaded pipelines configuration model or a new empty model if deserialization fails.</returns>
|
||||
/// <exception cref="ConfigLoadException">Thrown when the JSON cannot be parsed.</exception>
|
||||
public async Task<PipelinesConfigModel> LoadPipelinesAsync(string path, CancellationToken ct = default)
|
||||
{
|
||||
_logger?.LogInformation("Loading pipelines from {Path}", path);
|
||||
|
||||
try
|
||||
{
|
||||
var json = await _fileSystem.ReadAllTextAsync(path, ct);
|
||||
var config = JsonSerializer.Deserialize<PipelinesConfigModel>(json, JsonOptions);
|
||||
return config ?? new PipelinesConfigModel();
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new ConfigLoadException(path, $"Failed to parse pipelines.json: {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Saves the application settings configuration to the specified file path.
|
||||
/// </summary>
|
||||
@@ -92,15 +70,100 @@ public class ConfigFileService : IConfigFileService
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Saves the pipelines configuration to the specified file path.
|
||||
/// Loads a single pipeline from a pipeline.*.json file.
|
||||
/// </summary>
|
||||
/// <param name="path">The file path to save pipelines to.</param>
|
||||
/// <param name="config">The pipelines configuration model to save.</param>
|
||||
/// <param name="path">The file path to the pipeline file.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
public async Task SavePipelinesAsync(string path, PipelinesConfigModel config, CancellationToken ct = default)
|
||||
/// <returns>The loaded pipeline configuration.</returns>
|
||||
/// <exception cref="ConfigLoadException">Thrown when the JSON cannot be parsed.</exception>
|
||||
public async Task<EtlPipelineConfig> LoadPipelineAsync(string path, CancellationToken ct = default)
|
||||
{
|
||||
_logger?.LogInformation("Saving pipelines to {Path}", path);
|
||||
_logger?.LogDebug("Loading pipeline from {Path}", path);
|
||||
|
||||
try
|
||||
{
|
||||
var json = await _fileSystem.ReadAllTextAsync(path, ct);
|
||||
var config = JsonSerializer.Deserialize<EtlPipelineConfig>(json, JsonOptions);
|
||||
return config ?? throw new ConfigLoadException(path, "Pipeline file was empty or invalid.");
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new ConfigLoadException(path, $"Failed to parse pipeline file: {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Saves a single pipeline to a pipeline.*.json file.
|
||||
/// </summary>
|
||||
/// <param name="path">The file path to save the pipeline to.</param>
|
||||
/// <param name="config">The pipeline configuration to save.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
public async Task SavePipelineAsync(string path, EtlPipelineConfig config, CancellationToken ct = default)
|
||||
{
|
||||
_logger?.LogDebug("Saving pipeline to {Path}", path);
|
||||
var json = JsonSerializer.Serialize(config, JsonOptions);
|
||||
await _fileSystem.WriteAllTextAsync(path, json, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads all pipelines from a directory containing pipeline.*.json files.
|
||||
/// </summary>
|
||||
/// <param name="directory">The directory containing pipeline files.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
/// <returns>A dictionary of pipeline name to configuration.</returns>
|
||||
public async Task<Dictionary<string, EtlPipelineConfig>> LoadAllPipelinesAsync(string directory, CancellationToken ct = default)
|
||||
{
|
||||
_logger?.LogInformation("Loading all pipelines from {Directory}", directory);
|
||||
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (!_fileSystem.DirectoryExists(directory))
|
||||
{
|
||||
_logger?.LogWarning("Pipeline directory does not exist: {Directory}", directory);
|
||||
return pipelines;
|
||||
}
|
||||
|
||||
var files = await _fileSystem.GetFilesAsync(directory, "pipeline.*.json", ct);
|
||||
|
||||
foreach (var file in files)
|
||||
{
|
||||
try
|
||||
{
|
||||
var pipeline = await LoadPipelineAsync(file, ct);
|
||||
var fileName = _fileSystem.GetFileNameWithoutExtension(file);
|
||||
|
||||
// Extract name from "pipeline.{name}" format
|
||||
var name = fileName.StartsWith("pipeline.", StringComparison.OrdinalIgnoreCase)
|
||||
? fileName["pipeline.".Length..]
|
||||
: fileName;
|
||||
|
||||
// Ensure the pipeline name matches the filename
|
||||
if (string.IsNullOrEmpty(pipeline.Name))
|
||||
{
|
||||
pipeline.Name = name;
|
||||
}
|
||||
|
||||
pipelines[name] = pipeline;
|
||||
_logger?.LogDebug("Loaded pipeline: {Name}", name);
|
||||
}
|
||||
catch (ConfigLoadException ex)
|
||||
{
|
||||
_logger?.LogWarning(ex, "Failed to load pipeline from {File}", file);
|
||||
}
|
||||
}
|
||||
|
||||
_logger?.LogInformation("Loaded {Count} pipelines from {Directory}", pipelines.Count, directory);
|
||||
return pipelines;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a pipeline file.
|
||||
/// </summary>
|
||||
/// <param name="path">The file path to delete.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
public async Task DeletePipelineFileAsync(string path, CancellationToken ct = default)
|
||||
{
|
||||
_logger?.LogInformation("Deleting pipeline file: {Path}", path);
|
||||
await _fileSystem.DeleteFileAsync(path, ct);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Services;
|
||||
|
||||
@@ -15,14 +16,6 @@ public interface IConfigFileService
|
||||
/// <returns>The loaded configuration model or a new empty model if deserialization fails.</returns>
|
||||
Task<ConfigModel> LoadAppSettingsAsync(string path, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Loads the pipelines configuration from the specified file path.
|
||||
/// </summary>
|
||||
/// <param name="path">The file path to load pipelines from.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
/// <returns>The loaded pipelines configuration model or a new empty model if deserialization fails.</returns>
|
||||
Task<PipelinesConfigModel> LoadPipelinesAsync(string path, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Saves the application settings configuration to the specified file path.
|
||||
/// </summary>
|
||||
@@ -32,10 +25,33 @@ public interface IConfigFileService
|
||||
Task SaveAppSettingsAsync(string path, ConfigModel config, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Saves the pipelines configuration to the specified file path.
|
||||
/// Loads a single pipeline from a pipeline.*.json file.
|
||||
/// </summary>
|
||||
/// <param name="path">The file path to save pipelines to.</param>
|
||||
/// <param name="config">The pipelines configuration model to save.</param>
|
||||
/// <param name="path">The file path to the pipeline file.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
Task SavePipelinesAsync(string path, PipelinesConfigModel config, CancellationToken ct = default);
|
||||
/// <returns>The loaded pipeline configuration.</returns>
|
||||
Task<EtlPipelineConfig> LoadPipelineAsync(string path, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Saves a single pipeline to a pipeline.*.json file.
|
||||
/// </summary>
|
||||
/// <param name="path">The file path to save the pipeline to.</param>
|
||||
/// <param name="config">The pipeline configuration to save.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
Task SavePipelineAsync(string path, EtlPipelineConfig config, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Loads all pipelines from a directory containing pipeline.*.json files.
|
||||
/// </summary>
|
||||
/// <param name="directory">The directory containing pipeline files.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
/// <returns>A dictionary of pipeline name to configuration.</returns>
|
||||
Task<Dictionary<string, EtlPipelineConfig>> LoadAllPipelinesAsync(string directory, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a pipeline file.
|
||||
/// </summary>
|
||||
/// <param name="path">The file path to delete.</param>
|
||||
/// <param name="ct">Cancellation token for the operation.</param>
|
||||
Task DeletePipelineFileAsync(string path, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Services;
|
||||
|
||||
@@ -48,9 +49,17 @@ public interface IValidationService
|
||||
ValidationResult ValidateAppSettings(ConfigModel config);
|
||||
|
||||
/// <summary>
|
||||
/// Validates the pipelines configuration.
|
||||
/// Validates all pipelines in the dictionary.
|
||||
/// </summary>
|
||||
/// <param name="config">The pipelines configuration model to validate.</param>
|
||||
/// <param name="pipelines">Dictionary of pipeline name to configuration.</param>
|
||||
/// <returns>A validation result containing any errors or warnings found.</returns>
|
||||
ValidationResult ValidatePipelines(PipelinesConfigModel config);
|
||||
ValidationResult ValidatePipelines(Dictionary<string, EtlPipelineConfig> pipelines);
|
||||
|
||||
/// <summary>
|
||||
/// Validates a single pipeline configuration.
|
||||
/// </summary>
|
||||
/// <param name="pipeline">The pipeline configuration to validate.</param>
|
||||
/// <param name="name">The pipeline name (used in error messages).</param>
|
||||
/// <returns>A validation result containing any errors or warnings found.</returns>
|
||||
ValidationResult ValidatePipeline(EtlPipelineConfig pipeline, string name);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Services;
|
||||
|
||||
@@ -56,23 +57,49 @@ public class ValidationService : IValidationService
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates the pipelines configuration.
|
||||
/// Validates all pipelines in the dictionary.
|
||||
/// </summary>
|
||||
/// <param name="config">The pipelines configuration model to validate.</param>
|
||||
/// <param name="pipelines">Dictionary of pipeline name to configuration.</param>
|
||||
/// <returns>A validation result containing any errors or warnings found.</returns>
|
||||
public ValidationResult ValidatePipelines(PipelinesConfigModel config)
|
||||
public ValidationResult ValidatePipelines(Dictionary<string, EtlPipelineConfig> pipelines)
|
||||
{
|
||||
var result = new ValidationResult();
|
||||
|
||||
foreach (var (name, pipeline) in config.Pipelines)
|
||||
foreach (var (name, pipeline) in pipelines)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
result.AddError("Pipeline name cannot be empty");
|
||||
continue;
|
||||
}
|
||||
var pipelineResult = ValidatePipeline(pipeline, name);
|
||||
foreach (var error in pipelineResult.Errors)
|
||||
result.AddError(error);
|
||||
foreach (var warning in pipelineResult.Warnings)
|
||||
result.AddWarning(warning);
|
||||
}
|
||||
|
||||
// Source validation
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates a single pipeline configuration.
|
||||
/// </summary>
|
||||
/// <param name="pipeline">The pipeline configuration to validate.</param>
|
||||
/// <param name="name">The pipeline name (used in error messages).</param>
|
||||
/// <returns>A validation result containing any errors or warnings found.</returns>
|
||||
public ValidationResult ValidatePipeline(EtlPipelineConfig pipeline, string name)
|
||||
{
|
||||
var result = new ValidationResult();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
result.AddError("Pipeline name cannot be empty");
|
||||
return result;
|
||||
}
|
||||
|
||||
// Source validation
|
||||
if (pipeline.Source == null)
|
||||
{
|
||||
result.AddError($"Pipeline '{name}': Source is required");
|
||||
}
|
||||
else
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(pipeline.Source.Connection))
|
||||
{
|
||||
result.AddError($"Pipeline '{name}': Source.Connection is required");
|
||||
@@ -86,34 +113,55 @@ public class ValidationService : IValidationService
|
||||
{
|
||||
result.AddError($"Pipeline '{name}': Source.Query is required");
|
||||
}
|
||||
}
|
||||
|
||||
// Destination validation
|
||||
// Destination validation
|
||||
if (pipeline.Destination == null)
|
||||
{
|
||||
result.AddError($"Pipeline '{name}': Destination is required");
|
||||
}
|
||||
else
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(pipeline.Destination.Table))
|
||||
{
|
||||
result.AddError($"Pipeline '{name}': Destination.Table is required");
|
||||
}
|
||||
|
||||
if (pipeline.Destination.MatchColumns.Length == 0)
|
||||
if (pipeline.Destination.MatchColumns.Count == 0)
|
||||
{
|
||||
result.AddWarning($"Pipeline '{name}': No MatchColumns specified - all rows will be inserted");
|
||||
}
|
||||
}
|
||||
|
||||
// Schedule validation
|
||||
ValidateSchedule(result, name, "Mass", pipeline.Schedules.Mass, 60);
|
||||
ValidateSchedule(result, name, "Daily", pipeline.Schedules.Daily, 60);
|
||||
ValidateSchedule(result, name, "Hourly", pipeline.Schedules.Hourly, 15);
|
||||
// Schedule validation - must have at least one schedule unless manual-only
|
||||
if (!pipeline.IsManualOnly)
|
||||
{
|
||||
var hasSchedule = pipeline.MassSyncIntervalMinutes.HasValue ||
|
||||
pipeline.DailySyncIntervalMinutes.HasValue ||
|
||||
pipeline.HourlySyncIntervalMinutes.HasValue;
|
||||
|
||||
if (!hasSchedule)
|
||||
{
|
||||
result.AddWarning($"Pipeline '{name}': No sync schedule configured and not marked as manual-only");
|
||||
}
|
||||
|
||||
// Validate minimum intervals
|
||||
if (pipeline.MassSyncIntervalMinutes.HasValue && pipeline.MassSyncIntervalMinutes.Value < 60)
|
||||
{
|
||||
result.AddError($"Pipeline '{name}': Mass sync interval must be at least 60 minutes");
|
||||
}
|
||||
|
||||
if (pipeline.DailySyncIntervalMinutes.HasValue && pipeline.DailySyncIntervalMinutes.Value < 60)
|
||||
{
|
||||
result.AddError($"Pipeline '{name}': Daily sync interval must be at least 60 minutes");
|
||||
}
|
||||
|
||||
if (pipeline.HourlySyncIntervalMinutes.HasValue && pipeline.HourlySyncIntervalMinutes.Value < 15)
|
||||
{
|
||||
result.AddError($"Pipeline '{name}': Hourly sync interval must be at least 15 minutes");
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private void ValidateSchedule(ValidationResult result, string pipelineName, string scheduleName, ScheduleModel? schedule, int minInterval)
|
||||
{
|
||||
if (schedule == null) return;
|
||||
|
||||
if (schedule.Enabled && schedule.IntervalMinutes < minInterval)
|
||||
{
|
||||
result.AddError($"Pipeline '{pipelineName}': {scheduleName} schedule interval must be at least {minInterval} minutes");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+188
-55
@@ -1,6 +1,6 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.ConfigManager.Services;
|
||||
using JdeScoping.ConfigManager.ViewModels.PipelineSteps;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Windows.Input;
|
||||
|
||||
@@ -11,13 +11,13 @@ namespace JdeScoping.ConfigManager.ViewModels.Forms;
|
||||
/// </summary>
|
||||
public class PipelineEditorViewModel : ViewModelBase
|
||||
{
|
||||
private readonly PipelineModel _model;
|
||||
private readonly EtlPipelineConfig _model;
|
||||
private readonly Action _onChanged;
|
||||
private readonly IDialogService _dialogService;
|
||||
private PipelineStepViewModelBase? _selectedStep;
|
||||
private object? _selectedStepEditor;
|
||||
|
||||
public PipelineEditorViewModel(string name, PipelineModel model, IReadOnlyList<string> availableConnections, IDialogService dialogService, Action onChanged)
|
||||
public PipelineEditorViewModel(string name, EtlPipelineConfig model, IReadOnlyList<string> availableConnections, IDialogService dialogService, Action onChanged)
|
||||
{
|
||||
Name = name ?? throw new ArgumentNullException(nameof(name));
|
||||
_model = model ?? throw new ArgumentNullException(nameof(model));
|
||||
@@ -30,15 +30,6 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
Transformers = [];
|
||||
PostScripts = [];
|
||||
|
||||
// Initialize schedule view models
|
||||
_model.Schedules.Mass ??= new ScheduleModel();
|
||||
_model.Schedules.Daily ??= new ScheduleModel();
|
||||
_model.Schedules.Hourly ??= new ScheduleModel();
|
||||
|
||||
MassSchedule = new ScheduleFormViewModel(_model.Schedules.Mass, _onChanged);
|
||||
DailySchedule = new ScheduleFormViewModel(_model.Schedules.Daily, _onChanged);
|
||||
HourlySchedule = new ScheduleFormViewModel(_model.Schedules.Hourly, _onChanged);
|
||||
|
||||
// Build the pipeline steps from the model
|
||||
BuildPipelineSteps();
|
||||
|
||||
@@ -167,19 +158,164 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the mass schedule view model.
|
||||
/// Gets or sets whether the pipeline is enabled.
|
||||
/// </summary>
|
||||
public ScheduleFormViewModel MassSchedule { get; }
|
||||
public bool IsEnabled
|
||||
{
|
||||
get => _model.IsEnabled;
|
||||
set
|
||||
{
|
||||
if (_model.IsEnabled != value)
|
||||
{
|
||||
_model.IsEnabled = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the daily schedule view model.
|
||||
/// Gets or sets whether the pipeline is manual-only.
|
||||
/// </summary>
|
||||
public ScheduleFormViewModel DailySchedule { get; }
|
||||
public bool IsManualOnly
|
||||
{
|
||||
get => _model.IsManualOnly;
|
||||
set
|
||||
{
|
||||
if (_model.IsManualOnly != value)
|
||||
{
|
||||
_model.IsManualOnly = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the hourly schedule view model.
|
||||
/// Gets or sets whether mass sync is enabled.
|
||||
/// </summary>
|
||||
public ScheduleFormViewModel HourlySchedule { get; }
|
||||
public bool MassSyncEnabled
|
||||
{
|
||||
get => _model.MassSyncIntervalMinutes.HasValue;
|
||||
set
|
||||
{
|
||||
if (value && !_model.MassSyncIntervalMinutes.HasValue)
|
||||
{
|
||||
_model.MassSyncIntervalMinutes = 10080; // 1 week
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(MassSyncIntervalMinutes));
|
||||
_onChanged();
|
||||
}
|
||||
else if (!value && _model.MassSyncIntervalMinutes.HasValue)
|
||||
{
|
||||
_model.MassSyncIntervalMinutes = null;
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(MassSyncIntervalMinutes));
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the mass sync interval in minutes.
|
||||
/// </summary>
|
||||
public int MassSyncIntervalMinutes
|
||||
{
|
||||
get => _model.MassSyncIntervalMinutes ?? 10080;
|
||||
set
|
||||
{
|
||||
if (_model.MassSyncIntervalMinutes != value)
|
||||
{
|
||||
_model.MassSyncIntervalMinutes = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets whether daily sync is enabled.
|
||||
/// </summary>
|
||||
public bool DailySyncEnabled
|
||||
{
|
||||
get => _model.DailySyncIntervalMinutes.HasValue;
|
||||
set
|
||||
{
|
||||
if (value && !_model.DailySyncIntervalMinutes.HasValue)
|
||||
{
|
||||
_model.DailySyncIntervalMinutes = 1440; // 1 day
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(DailySyncIntervalMinutes));
|
||||
_onChanged();
|
||||
}
|
||||
else if (!value && _model.DailySyncIntervalMinutes.HasValue)
|
||||
{
|
||||
_model.DailySyncIntervalMinutes = null;
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(DailySyncIntervalMinutes));
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the daily sync interval in minutes.
|
||||
/// </summary>
|
||||
public int DailySyncIntervalMinutes
|
||||
{
|
||||
get => _model.DailySyncIntervalMinutes ?? 1440;
|
||||
set
|
||||
{
|
||||
if (_model.DailySyncIntervalMinutes != value)
|
||||
{
|
||||
_model.DailySyncIntervalMinutes = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets whether hourly sync is enabled.
|
||||
/// </summary>
|
||||
public bool HourlySyncEnabled
|
||||
{
|
||||
get => _model.HourlySyncIntervalMinutes.HasValue;
|
||||
set
|
||||
{
|
||||
if (value && !_model.HourlySyncIntervalMinutes.HasValue)
|
||||
{
|
||||
_model.HourlySyncIntervalMinutes = 60; // 1 hour
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(HourlySyncIntervalMinutes));
|
||||
_onChanged();
|
||||
}
|
||||
else if (!value && _model.HourlySyncIntervalMinutes.HasValue)
|
||||
{
|
||||
_model.HourlySyncIntervalMinutes = null;
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(HourlySyncIntervalMinutes));
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the hourly sync interval in minutes.
|
||||
/// </summary>
|
||||
public int HourlySyncIntervalMinutes
|
||||
{
|
||||
get => _model.HourlySyncIntervalMinutes ?? 60;
|
||||
set
|
||||
{
|
||||
if (_model.HourlySyncIntervalMinutes != value)
|
||||
{
|
||||
_model.HourlySyncIntervalMinutes = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Commands
|
||||
public ICommand AddPreScriptCommand { get; }
|
||||
@@ -211,16 +347,13 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
{
|
||||
// Pre-scripts
|
||||
PreScripts.Clear();
|
||||
if (_model.PreScripts != null)
|
||||
foreach (var script in _model.PreScripts)
|
||||
{
|
||||
foreach (var script in _model.PreScripts)
|
||||
PreScripts.Add(new PreScriptStepViewModel(script, () =>
|
||||
{
|
||||
PreScripts.Add(new PreScriptStepViewModel(script, () =>
|
||||
{
|
||||
SyncPreScriptsToModel();
|
||||
_onChanged();
|
||||
}));
|
||||
}
|
||||
SyncPreScriptsToModel();
|
||||
_onChanged();
|
||||
}));
|
||||
}
|
||||
|
||||
// Source
|
||||
@@ -231,18 +364,15 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
|
||||
// Transformers
|
||||
Transformers.Clear();
|
||||
if (_model.Transformers != null)
|
||||
foreach (var transform in _model.Transforms)
|
||||
{
|
||||
foreach (var transformer in _model.Transformers)
|
||||
var vm = TransformerFactory.Create(transform, () =>
|
||||
{
|
||||
var vm = TransformerFactory.Create(transformer, () =>
|
||||
{
|
||||
SyncTransformersToModel();
|
||||
_onChanged();
|
||||
});
|
||||
if (vm != null)
|
||||
Transformers.Add(vm);
|
||||
}
|
||||
SyncTransformersToModel();
|
||||
_onChanged();
|
||||
});
|
||||
if (vm != null)
|
||||
Transformers.Add(vm);
|
||||
}
|
||||
|
||||
// Destination
|
||||
@@ -253,16 +383,13 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
|
||||
// Post-scripts
|
||||
PostScripts.Clear();
|
||||
if (_model.PostScripts != null)
|
||||
foreach (var script in _model.PostScripts)
|
||||
{
|
||||
foreach (var script in _model.PostScripts)
|
||||
PostScripts.Add(new PostScriptStepViewModel(script, () =>
|
||||
{
|
||||
PostScripts.Add(new PostScriptStepViewModel(script, () =>
|
||||
{
|
||||
SyncPostScriptsToModel();
|
||||
_onChanged();
|
||||
}));
|
||||
}
|
||||
SyncPostScriptsToModel();
|
||||
_onChanged();
|
||||
}));
|
||||
}
|
||||
|
||||
OnPropertyChanged(nameof(AllSteps));
|
||||
@@ -277,7 +404,7 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
|
||||
private void AddPreScript()
|
||||
{
|
||||
var step = new PreScriptStepViewModel(string.Empty, () =>
|
||||
var step = new PreScriptStepViewModel(() =>
|
||||
{
|
||||
SyncPreScriptsToModel();
|
||||
_onChanged();
|
||||
@@ -330,7 +457,7 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
|
||||
private void AddPostScript()
|
||||
{
|
||||
var step = new PostScriptStepViewModel(string.Empty, () =>
|
||||
var step = new PostScriptStepViewModel(() =>
|
||||
{
|
||||
SyncPostScriptsToModel();
|
||||
_onChanged();
|
||||
@@ -477,23 +604,29 @@ public class PipelineEditorViewModel : ViewModelBase
|
||||
|
||||
private void SyncPreScriptsToModel()
|
||||
{
|
||||
_model.PreScripts = PreScripts.Count > 0
|
||||
? PreScripts.Select(s => s.Script).ToArray()
|
||||
: null;
|
||||
_model.PreScripts.Clear();
|
||||
foreach (var script in PreScripts)
|
||||
{
|
||||
_model.PreScripts.Add(script.ToModel());
|
||||
}
|
||||
}
|
||||
|
||||
private void SyncTransformersToModel()
|
||||
{
|
||||
_model.Transformers = Transformers.Count > 0
|
||||
? Transformers.Select(t => t.ToModel()).ToList()
|
||||
: null;
|
||||
_model.Transforms.Clear();
|
||||
foreach (var transformer in Transformers)
|
||||
{
|
||||
_model.Transforms.Add(transformer.ToModel());
|
||||
}
|
||||
}
|
||||
|
||||
private void SyncPostScriptsToModel()
|
||||
{
|
||||
_model.PostScripts = PostScripts.Count > 0
|
||||
? PostScripts.Select(s => s.Script).ToArray()
|
||||
: null;
|
||||
_model.PostScripts.Clear();
|
||||
foreach (var script in PostScripts)
|
||||
{
|
||||
_model.PostScripts.Add(script.ToModel());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
|
||||
namespace JdeScoping.ConfigManager.ViewModels.Forms;
|
||||
|
||||
/// <summary>
|
||||
/// ViewModel for editing a pipeline configuration.
|
||||
/// </summary>
|
||||
public class PipelineFormViewModel : ViewModelBase
|
||||
{
|
||||
private readonly PipelineModel _model;
|
||||
private readonly Action _onChanged;
|
||||
|
||||
public PipelineFormViewModel(string name, PipelineModel model, Action onChanged)
|
||||
{
|
||||
Name = name ?? throw new ArgumentNullException(nameof(name));
|
||||
_model = model ?? throw new ArgumentNullException(nameof(model));
|
||||
_onChanged = onChanged ?? throw new ArgumentNullException(nameof(onChanged));
|
||||
|
||||
// Initialize schedule view models
|
||||
_model.Schedules.Mass ??= new ScheduleModel();
|
||||
_model.Schedules.Daily ??= new ScheduleModel();
|
||||
_model.Schedules.Hourly ??= new ScheduleModel();
|
||||
|
||||
MassSchedule = new ScheduleFormViewModel(_model.Schedules.Mass, _onChanged);
|
||||
DailySchedule = new ScheduleFormViewModel(_model.Schedules.Daily, _onChanged);
|
||||
HourlySchedule = new ScheduleFormViewModel(_model.Schedules.Hourly, _onChanged);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the pipeline name.
|
||||
/// </summary>
|
||||
public string Name { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the source connection name.
|
||||
/// </summary>
|
||||
public string Connection
|
||||
{
|
||||
get => _model.Source.Connection;
|
||||
set
|
||||
{
|
||||
if (_model.Source.Connection != value)
|
||||
{
|
||||
_model.Source.Connection = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the source query.
|
||||
/// </summary>
|
||||
public string Query
|
||||
{
|
||||
get => _model.Source.Query;
|
||||
set
|
||||
{
|
||||
if (_model.Source.Query != value)
|
||||
{
|
||||
_model.Source.Query = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the optional mass query.
|
||||
/// </summary>
|
||||
public string? MassQuery
|
||||
{
|
||||
get => _model.Source.MassQuery;
|
||||
set
|
||||
{
|
||||
if (_model.Source.MassQuery != value)
|
||||
{
|
||||
_model.Source.MassQuery = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the destination table name.
|
||||
/// </summary>
|
||||
public string DestinationTable
|
||||
{
|
||||
get => _model.Destination.Table;
|
||||
set
|
||||
{
|
||||
if (_model.Destination.Table != value)
|
||||
{
|
||||
_model.Destination.Table = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the match columns as newline-separated text.
|
||||
/// </summary>
|
||||
public string MatchColumnsText
|
||||
{
|
||||
get => string.Join("\n", _model.Destination.MatchColumns);
|
||||
set
|
||||
{
|
||||
var columns = value.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (!_model.Destination.MatchColumns.SequenceEqual(columns))
|
||||
{
|
||||
_model.Destination.MatchColumns = columns;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the exclude from update columns as newline-separated text.
|
||||
/// </summary>
|
||||
public string ExcludeFromUpdateText
|
||||
{
|
||||
get => string.Join("\n", _model.Destination.ExcludeFromUpdate);
|
||||
set
|
||||
{
|
||||
var columns = value.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (!_model.Destination.ExcludeFromUpdate.SequenceEqual(columns))
|
||||
{
|
||||
_model.Destination.ExcludeFromUpdate = columns;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the post scripts as newline-separated text.
|
||||
/// </summary>
|
||||
public string PostScriptsText
|
||||
{
|
||||
get => _model.PostScripts != null ? string.Join("\n", _model.PostScripts) : string.Empty;
|
||||
set
|
||||
{
|
||||
var scripts = string.IsNullOrWhiteSpace(value)
|
||||
? null
|
||||
: value.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (_model.PostScripts?.SequenceEqual(scripts ?? []) != true)
|
||||
{
|
||||
_model.PostScripts = scripts;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the mass schedule view model.
|
||||
/// </summary>
|
||||
public ScheduleFormViewModel MassSchedule { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the daily schedule view model.
|
||||
/// </summary>
|
||||
public ScheduleFormViewModel DailySchedule { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the hourly schedule view model.
|
||||
/// </summary>
|
||||
public ScheduleFormViewModel HourlySchedule { get; }
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
|
||||
namespace JdeScoping.ConfigManager.ViewModels.Forms;
|
||||
|
||||
/// <summary>
|
||||
/// ViewModel for editing a schedule configuration.
|
||||
/// </summary>
|
||||
public class ScheduleFormViewModel : ViewModelBase
|
||||
{
|
||||
private readonly ScheduleModel _model;
|
||||
private readonly Action _onChanged;
|
||||
|
||||
public ScheduleFormViewModel(ScheduleModel model, Action onChanged)
|
||||
{
|
||||
_model = model ?? throw new ArgumentNullException(nameof(model));
|
||||
_onChanged = onChanged ?? throw new ArgumentNullException(nameof(onChanged));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets whether this schedule is enabled.
|
||||
/// </summary>
|
||||
public bool Enabled
|
||||
{
|
||||
get => _model.Enabled;
|
||||
set
|
||||
{
|
||||
if (_model.Enabled != value)
|
||||
{
|
||||
_model.Enabled = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the interval in minutes.
|
||||
/// </summary>
|
||||
public int IntervalMinutes
|
||||
{
|
||||
get => _model.IntervalMinutes;
|
||||
set
|
||||
{
|
||||
if (_model.IntervalMinutes != value)
|
||||
{
|
||||
_model.IntervalMinutes = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets whether to purge before sync.
|
||||
/// </summary>
|
||||
public bool PrePurge
|
||||
{
|
||||
get => _model.PrePurge;
|
||||
set
|
||||
{
|
||||
if (_model.PrePurge != value)
|
||||
{
|
||||
_model.PrePurge = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets whether to reindex after sync.
|
||||
/// </summary>
|
||||
public bool ReIndex
|
||||
{
|
||||
get => _model.ReIndex;
|
||||
set
|
||||
{
|
||||
if (_model.ReIndex != value)
|
||||
{
|
||||
_model.ReIndex = value;
|
||||
OnPropertyChanged();
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ using JdeScoping.ConfigManager.Services;
|
||||
using JdeScoping.ConfigManager.Services.SecureStore;
|
||||
using JdeScoping.ConfigManager.ViewModels.Dialogs;
|
||||
using JdeScoping.ConfigManager.ViewModels.Forms;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace JdeScoping.ConfigManager.ViewModels;
|
||||
@@ -36,7 +37,7 @@ public class MainWindowViewModel : ViewModelBase
|
||||
private object? _selectedFormViewModel;
|
||||
|
||||
private ConfigModel? _appSettings;
|
||||
private PipelinesConfigModel? _pipelines;
|
||||
private Dictionary<string, EtlPipelineConfig>? _pipelines;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the currently loaded configuration folder path.
|
||||
@@ -423,13 +424,17 @@ public class MainWindowViewModel : ViewModelBase
|
||||
var appSettingsPath = Path.Combine(folderPath, "appsettings.json");
|
||||
_appSettings = await _configFileService.LoadAppSettingsAsync(appSettingsPath);
|
||||
|
||||
// Use config-driven pipeline path
|
||||
var pipelinesConfigPath = _appSettings?.Pipelines?.ConfigPath ?? "Pipelines/pipelines.json";
|
||||
var pipelinesPath = Path.Combine(folderPath, pipelinesConfigPath);
|
||||
// Load pipelines from directory containing pipeline.*.json files
|
||||
var pipelinesDirectory = Path.Combine(folderPath,
|
||||
_appSettings?.Pipelines?.ConfigDirectory ?? "Pipelines");
|
||||
|
||||
if (File.Exists(pipelinesPath))
|
||||
if (Directory.Exists(pipelinesDirectory))
|
||||
{
|
||||
_pipelines = await _configFileService.LoadPipelinesAsync(pipelinesPath);
|
||||
_pipelines = await _configFileService.LoadAllPipelinesAsync(pipelinesDirectory);
|
||||
}
|
||||
else
|
||||
{
|
||||
_pipelines = new Dictionary<string, EtlPipelineConfig>(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
// Initialize SecureStore (auto-create if needed, open, sync required keys)
|
||||
@@ -468,7 +473,7 @@ public class MainWindowViewModel : ViewModelBase
|
||||
var pipelinesFolder = new TreeNodeViewModel("Pipelines", "⚡", TreeNodeType.Folder) { IsExpanded = true };
|
||||
if (_pipelines != null)
|
||||
{
|
||||
foreach (var (name, _) in _pipelines.Pipelines)
|
||||
foreach (var name in _pipelines.Keys.OrderBy(k => k))
|
||||
{
|
||||
pipelinesFolder.Children.Add(new TreeNodeViewModel(name, "📦", TreeNodeType.Pipeline) { SectionKey = name });
|
||||
}
|
||||
@@ -574,7 +579,7 @@ public class MainWindowViewModel : ViewModelBase
|
||||
_dialogService,
|
||||
_connectionTestService),
|
||||
_ when _selectedNode.NodeType == TreeNodeType.Pipeline && _pipelines != null && _dialogService != null
|
||||
=> _pipelines.Pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline)
|
||||
=> _pipelines.TryGetValue(_selectedNode.SectionKey!, out var pipeline)
|
||||
? new PipelineEditorViewModel(_selectedNode.SectionKey!, pipeline, GetAvailableConnections(), _dialogService, MarkAsChanged)
|
||||
: null,
|
||||
_ => null
|
||||
@@ -659,8 +664,8 @@ public class MainWindowViewModel : ViewModelBase
|
||||
/// Loads configuration for testing purposes.
|
||||
/// </summary>
|
||||
/// <param name="appSettings">The application settings configuration model.</param>
|
||||
/// <param name="pipelines">The pipelines configuration model.</param>
|
||||
public void LoadConfigForTesting(ConfigModel? appSettings, PipelinesConfigModel? pipelines)
|
||||
/// <param name="pipelines">The pipelines dictionary.</param>
|
||||
public void LoadConfigForTesting(ConfigModel? appSettings, Dictionary<string, EtlPipelineConfig>? pipelines)
|
||||
{
|
||||
_appSettings = appSettings;
|
||||
_pipelines = pipelines;
|
||||
@@ -687,16 +692,19 @@ public class MainWindowViewModel : ViewModelBase
|
||||
// Save appsettings
|
||||
await _configFileService.SaveAppSettingsAsync(appSettingsPath, _appSettings);
|
||||
|
||||
// Save pipelines if loaded
|
||||
// Save each pipeline to its own file
|
||||
if (_pipelines != null)
|
||||
{
|
||||
var pipelinesConfigPath = _appSettings?.Pipelines?.ConfigPath ?? "Pipelines/pipelines.json";
|
||||
var pipelinesPath = Path.Combine(ConfigFolderPath, pipelinesConfigPath);
|
||||
if (File.Exists(pipelinesPath))
|
||||
var pipelinesDirectory = Path.Combine(ConfigFolderPath,
|
||||
_appSettings?.Pipelines?.ConfigDirectory ?? "Pipelines");
|
||||
|
||||
Directory.CreateDirectory(pipelinesDirectory);
|
||||
|
||||
foreach (var (name, pipeline) in _pipelines)
|
||||
{
|
||||
await _backupService.CreateBackupAsync(pipelinesPath);
|
||||
var filePath = Path.Combine(pipelinesDirectory, $"pipeline.{name}.json");
|
||||
await _configFileService.SavePipelineAsync(filePath, pipeline);
|
||||
}
|
||||
await _configFileService.SavePipelinesAsync(pipelinesPath, _pipelines);
|
||||
}
|
||||
|
||||
HasUnsavedChanges = false;
|
||||
@@ -834,22 +842,23 @@ public class MainWindowViewModel : ViewModelBase
|
||||
return;
|
||||
|
||||
// Check for duplicate
|
||||
if (_pipelines.Pipelines.ContainsKey(name))
|
||||
if (_pipelines.ContainsKey(name))
|
||||
{
|
||||
await _dialogService.ShowMessageAsync("Error",
|
||||
$"Pipeline '{name}' already exists.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create default pipeline model
|
||||
var pipeline = new PipelineModel
|
||||
// Create default pipeline using EtlPipelineConfig
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new PipelineSource { Connection = "lotfinder", Query = "" },
|
||||
Destination = new PipelineDestination { Table = name },
|
||||
Schedules = new PipelineSchedules()
|
||||
Name = name,
|
||||
IsEnabled = true,
|
||||
Source = new SourceElement { Connection = "lotfinder", Query = "" },
|
||||
Destination = new DestinationElement { Table = name }
|
||||
};
|
||||
|
||||
_pipelines.Pipelines[name] = pipeline;
|
||||
_pipelines[name] = pipeline;
|
||||
|
||||
// Add tree node
|
||||
var pipelinesFolder = TreeNodes.FirstOrDefault(n =>
|
||||
@@ -875,7 +884,8 @@ public class MainWindowViewModel : ViewModelBase
|
||||
{
|
||||
if (_selectedNode?.NodeType != TreeNodeType.Pipeline ||
|
||||
_pipelines == null ||
|
||||
_dialogService == null)
|
||||
_dialogService == null ||
|
||||
_appSettings == null)
|
||||
return;
|
||||
|
||||
var name = _selectedNode.SectionKey!;
|
||||
@@ -888,7 +898,16 @@ public class MainWindowViewModel : ViewModelBase
|
||||
return;
|
||||
|
||||
// Remove from model
|
||||
_pipelines.Pipelines.Remove(name);
|
||||
_pipelines.Remove(name);
|
||||
|
||||
// Delete the pipeline file
|
||||
var pipelinesDirectory = Path.Combine(ConfigFolderPath,
|
||||
_appSettings.Pipelines?.ConfigDirectory ?? "Pipelines");
|
||||
var filePath = Path.Combine(pipelinesDirectory, $"pipeline.{name}.json");
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
await _configFileService.DeletePipelineFileAsync(filePath);
|
||||
}
|
||||
|
||||
// Remove tree node
|
||||
var pipelinesFolder = TreeNodes.FirstOrDefault(n =>
|
||||
|
||||
+5
-50
@@ -1,24 +1,15 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps;
|
||||
|
||||
/// <summary>
|
||||
/// Destination type for the pipeline.
|
||||
/// </summary>
|
||||
public enum DestinationType
|
||||
{
|
||||
BulkImport,
|
||||
BulkMerge
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// View model for the destination step in a pipeline.
|
||||
/// </summary>
|
||||
public class DestinationStepViewModel : PipelineStepViewModelBase
|
||||
{
|
||||
private readonly PipelineDestination _model;
|
||||
private readonly DestinationElement _model;
|
||||
|
||||
public DestinationStepViewModel(PipelineDestination model, Action onChanged) : base(onChanged)
|
||||
public DestinationStepViewModel(DestinationElement model, Action onChanged) : base(onChanged)
|
||||
{
|
||||
_model = model ?? throw new ArgumentNullException(nameof(model));
|
||||
}
|
||||
@@ -28,40 +19,6 @@ public class DestinationStepViewModel : PipelineStepViewModelBase
|
||||
public override string Icon => ""; // mdi-database
|
||||
public override string Summary => !string.IsNullOrEmpty(Table) ? $"→ {Table}" : "(no table)";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the destination type (BulkImport or BulkMerge).
|
||||
/// </summary>
|
||||
public DestinationType Type
|
||||
{
|
||||
get => _model.Type?.Equals("BulkImport", StringComparison.OrdinalIgnoreCase) == true
|
||||
? DestinationType.BulkImport
|
||||
: DestinationType.BulkMerge;
|
||||
set
|
||||
{
|
||||
var typeStr = value == DestinationType.BulkImport ? "BulkImport" : "BulkMerge";
|
||||
if (_model.Type != typeStr)
|
||||
{
|
||||
_model.Type = typeStr;
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(IsBulkMerge));
|
||||
OnPropertyChanged(nameof(TypeDescription));
|
||||
NotifyChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether the destination type is BulkMerge (shows match columns).
|
||||
/// </summary>
|
||||
public bool IsBulkMerge => Type == DestinationType.BulkMerge;
|
||||
|
||||
/// <summary>
|
||||
/// Gets a description of the current type.
|
||||
/// </summary>
|
||||
public string TypeDescription => Type == DestinationType.BulkImport
|
||||
? "Truncate table and bulk load all data"
|
||||
: "Merge data using match columns (upsert)";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the destination table name.
|
||||
/// </summary>
|
||||
@@ -82,14 +39,13 @@ public class DestinationStepViewModel : PipelineStepViewModelBase
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the match columns as newline-separated text.
|
||||
/// Only used for BulkMerge type.
|
||||
/// </summary>
|
||||
public string MatchColumnsText
|
||||
{
|
||||
get => string.Join("\n", _model.MatchColumns);
|
||||
set
|
||||
{
|
||||
var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList();
|
||||
if (!_model.MatchColumns.SequenceEqual(columns))
|
||||
{
|
||||
_model.MatchColumns = columns;
|
||||
@@ -101,14 +57,13 @@ public class DestinationStepViewModel : PipelineStepViewModelBase
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the columns to exclude from updates as newline-separated text.
|
||||
/// Only used for BulkMerge type.
|
||||
/// </summary>
|
||||
public string ExcludeFromUpdateText
|
||||
{
|
||||
get => string.Join("\n", _model.ExcludeFromUpdate);
|
||||
set
|
||||
{
|
||||
var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
var columns = (value ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList();
|
||||
if (!_model.ExcludeFromUpdate.SequenceEqual(columns))
|
||||
{
|
||||
_model.ExcludeFromUpdate = columns;
|
||||
|
||||
+77
-12
@@ -1,4 +1,5 @@
|
||||
using System.Windows.Input;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps;
|
||||
|
||||
@@ -70,34 +71,66 @@ public abstract class PipelineStepViewModelBase : ViewModelBase
|
||||
/// </summary>
|
||||
public class PreScriptStepViewModel : PipelineStepViewModelBase
|
||||
{
|
||||
private string _script;
|
||||
private readonly ScriptElement _model;
|
||||
|
||||
public PreScriptStepViewModel(string script, Action onChanged) : base(onChanged)
|
||||
public PreScriptStepViewModel(ScriptElement model, Action onChanged) : base(onChanged)
|
||||
{
|
||||
_script = script ?? string.Empty;
|
||||
_model = model ?? throw new ArgumentNullException(nameof(model));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new pre-script with default values.
|
||||
/// </summary>
|
||||
public PreScriptStepViewModel(Action onChanged) : base(onChanged)
|
||||
{
|
||||
_model = new ScriptElement { Connection = "lotfinder", Script = string.Empty };
|
||||
}
|
||||
|
||||
public override PipelineStepType StepType => PipelineStepType.PreScript;
|
||||
public override string DisplayName => "Pre-Script";
|
||||
public override string Icon => ""; // mdi-script-text
|
||||
public override string Summary => TruncateScript(_script);
|
||||
public override string Summary => TruncateScript(_model.Script);
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the connection for script execution.
|
||||
/// </summary>
|
||||
public string Connection
|
||||
{
|
||||
get => _model.Connection;
|
||||
set
|
||||
{
|
||||
if (_model.Connection != value)
|
||||
{
|
||||
_model.Connection = value ?? "lotfinder";
|
||||
OnPropertyChanged();
|
||||
NotifyChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the SQL script content.
|
||||
/// </summary>
|
||||
public string Script
|
||||
{
|
||||
get => _script;
|
||||
get => _model.Script;
|
||||
set
|
||||
{
|
||||
if (SetProperty(ref _script, value ?? string.Empty))
|
||||
if (_model.Script != value)
|
||||
{
|
||||
_model.Script = value ?? string.Empty;
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(Summary));
|
||||
NotifyChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the underlying model.
|
||||
/// </summary>
|
||||
public ScriptElement ToModel() => _model;
|
||||
|
||||
private static string TruncateScript(string script)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(script)) return "(empty)";
|
||||
@@ -111,34 +144,66 @@ public class PreScriptStepViewModel : PipelineStepViewModelBase
|
||||
/// </summary>
|
||||
public class PostScriptStepViewModel : PipelineStepViewModelBase
|
||||
{
|
||||
private string _script;
|
||||
private readonly ScriptElement _model;
|
||||
|
||||
public PostScriptStepViewModel(string script, Action onChanged) : base(onChanged)
|
||||
public PostScriptStepViewModel(ScriptElement model, Action onChanged) : base(onChanged)
|
||||
{
|
||||
_script = script ?? string.Empty;
|
||||
_model = model ?? throw new ArgumentNullException(nameof(model));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new post-script with default values.
|
||||
/// </summary>
|
||||
public PostScriptStepViewModel(Action onChanged) : base(onChanged)
|
||||
{
|
||||
_model = new ScriptElement { Connection = "lotfinder", Script = string.Empty };
|
||||
}
|
||||
|
||||
public override PipelineStepType StepType => PipelineStepType.PostScript;
|
||||
public override string DisplayName => "Post-Script";
|
||||
public override string Icon => ""; // mdi-script-text
|
||||
public override string Summary => TruncateScript(_script);
|
||||
public override string Summary => TruncateScript(_model.Script);
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the connection for script execution.
|
||||
/// </summary>
|
||||
public string Connection
|
||||
{
|
||||
get => _model.Connection;
|
||||
set
|
||||
{
|
||||
if (_model.Connection != value)
|
||||
{
|
||||
_model.Connection = value ?? "lotfinder";
|
||||
OnPropertyChanged();
|
||||
NotifyChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the SQL script content.
|
||||
/// </summary>
|
||||
public string Script
|
||||
{
|
||||
get => _script;
|
||||
get => _model.Script;
|
||||
set
|
||||
{
|
||||
if (SetProperty(ref _script, value ?? string.Empty))
|
||||
if (_model.Script != value)
|
||||
{
|
||||
_model.Script = value ?? string.Empty;
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(Summary));
|
||||
NotifyChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the underlying model.
|
||||
/// </summary>
|
||||
public ScriptElement ToModel() => _model;
|
||||
|
||||
private static string TruncateScript(string script)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(script)) return "(empty)";
|
||||
|
||||
+28
-85
@@ -1,26 +1,17 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Windows.Input;
|
||||
|
||||
namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps;
|
||||
|
||||
/// <summary>
|
||||
/// Source type for the pipeline.
|
||||
/// </summary>
|
||||
public enum SourceType
|
||||
{
|
||||
Database,
|
||||
File
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// View model for the source step in a pipeline.
|
||||
/// </summary>
|
||||
public class SourceStepViewModel : PipelineStepViewModelBase
|
||||
{
|
||||
private readonly PipelineSource _model;
|
||||
private readonly SourceElement _model;
|
||||
|
||||
public SourceStepViewModel(PipelineSource model, IReadOnlyList<string> availableConnections, Action onChanged) : base(onChanged)
|
||||
public SourceStepViewModel(SourceElement model, IReadOnlyList<string> availableConnections, Action onChanged) : base(onChanged)
|
||||
{
|
||||
_model = model ?? throw new ArgumentNullException(nameof(model));
|
||||
AvailableConnections = availableConnections ?? [];
|
||||
@@ -44,54 +35,8 @@ public class SourceStepViewModel : PipelineStepViewModelBase
|
||||
|
||||
public override PipelineStepType StepType => PipelineStepType.Source;
|
||||
public override string DisplayName => "Source";
|
||||
public override string Icon => IsFileSource ? "" : ""; // mdi-file vs mdi-database
|
||||
public override string Summary => IsFileSource
|
||||
? $"File: {System.IO.Path.GetFileName(FileName) ?? "(none)"}"
|
||||
: $"{Connection}: {TruncateQuery(Query)}";
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether this is a file-based source.
|
||||
/// </summary>
|
||||
public bool IsFileSource => !string.IsNullOrEmpty(_model.FileName);
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether this is a database source.
|
||||
/// </summary>
|
||||
public bool IsDatabaseSource => !IsFileSource;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the source type (Database or File).
|
||||
/// </summary>
|
||||
public SourceType SourceType
|
||||
{
|
||||
get => IsFileSource ? SourceType.File : SourceType.Database;
|
||||
set
|
||||
{
|
||||
if (value == SourceType.File && !IsFileSource)
|
||||
{
|
||||
// Switching to file source
|
||||
_model.FileName = string.Empty;
|
||||
_model.Connection = string.Empty;
|
||||
_model.Query = string.Empty;
|
||||
_model.MassQuery = null;
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(IsFileSource));
|
||||
OnPropertyChanged(nameof(IsDatabaseSource));
|
||||
OnPropertyChanged(nameof(Summary));
|
||||
NotifyChanged();
|
||||
}
|
||||
else if (value == SourceType.Database && IsFileSource)
|
||||
{
|
||||
// Switching to database source
|
||||
_model.FileName = null;
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(IsFileSource));
|
||||
OnPropertyChanged(nameof(IsDatabaseSource));
|
||||
OnPropertyChanged(nameof(Summary));
|
||||
NotifyChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
public override string Icon => ""; // mdi-database
|
||||
public override string Summary => $"{Connection}: {TruncateQuery(Query)}";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the source database connection name.
|
||||
@@ -146,24 +91,6 @@ public class SourceStepViewModel : PipelineStepViewModelBase
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the file name for file-based sources.
|
||||
/// </summary>
|
||||
public string? FileName
|
||||
{
|
||||
get => _model.FileName;
|
||||
set
|
||||
{
|
||||
if (_model.FileName != value)
|
||||
{
|
||||
_model.FileName = value;
|
||||
OnPropertyChanged();
|
||||
OnPropertyChanged(nameof(Summary));
|
||||
NotifyChanged();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the collection of query parameters.
|
||||
/// </summary>
|
||||
@@ -180,7 +107,7 @@ public class SourceStepViewModel : PipelineStepViewModelBase
|
||||
public void AddParameter()
|
||||
{
|
||||
var key = $"param{Parameters.Count + 1}";
|
||||
var param = new ParameterDefinition { Name = key };
|
||||
var param = new ParameterElement { Name = key, Source = "offset" };
|
||||
var vm = new ParameterViewModel(key, param, () =>
|
||||
{
|
||||
SyncParametersToModel();
|
||||
@@ -228,15 +155,17 @@ public class ParameterViewModel : ViewModelBase
|
||||
private string _key;
|
||||
private string _name;
|
||||
private string? _format;
|
||||
private string? _source;
|
||||
private string _source;
|
||||
private string? _value;
|
||||
private readonly Action _onChanged;
|
||||
|
||||
public ParameterViewModel(string key, ParameterDefinition model, Action onChanged)
|
||||
public ParameterViewModel(string key, ParameterElement model, Action onChanged)
|
||||
{
|
||||
_key = key;
|
||||
_name = model.Name;
|
||||
_format = model.Format;
|
||||
_source = model.Source;
|
||||
_value = model.Value;
|
||||
_onChanged = onChanged;
|
||||
}
|
||||
|
||||
@@ -282,12 +211,25 @@ public class ParameterViewModel : ViewModelBase
|
||||
/// <summary>
|
||||
/// Gets or sets the parameter source (e.g., offset, static).
|
||||
/// </summary>
|
||||
public string? Source
|
||||
public string Source
|
||||
{
|
||||
get => _source;
|
||||
set
|
||||
{
|
||||
if (SetProperty(ref _source, value))
|
||||
if (SetProperty(ref _source, value ?? "offset"))
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the static value (if source is not offset).
|
||||
/// </summary>
|
||||
public string? Value
|
||||
{
|
||||
get => _value;
|
||||
set
|
||||
{
|
||||
if (SetProperty(ref _value, value))
|
||||
_onChanged();
|
||||
}
|
||||
}
|
||||
@@ -295,10 +237,11 @@ public class ParameterViewModel : ViewModelBase
|
||||
/// <summary>
|
||||
/// Converts this view model back to a model.
|
||||
/// </summary>
|
||||
public ParameterDefinition ToModel() => new()
|
||||
public ParameterElement ToModel() => new()
|
||||
{
|
||||
Name = _name,
|
||||
Format = _format,
|
||||
Source = _source
|
||||
Source = _source,
|
||||
Value = _value
|
||||
};
|
||||
}
|
||||
|
||||
+134
-47
@@ -1,5 +1,6 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Windows.Input;
|
||||
|
||||
@@ -10,6 +11,11 @@ namespace JdeScoping.ConfigManager.ViewModels.PipelineSteps;
|
||||
/// </summary>
|
||||
public abstract class TransformerStepViewModelBase : PipelineStepViewModelBase
|
||||
{
|
||||
protected static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
protected TransformerStepViewModelBase(Action onChanged) : base(onChanged)
|
||||
{
|
||||
}
|
||||
@@ -23,9 +29,19 @@ public abstract class TransformerStepViewModelBase : PipelineStepViewModelBase
|
||||
public abstract string TransformerType { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Converts this view model back to a model.
|
||||
/// Converts this view model back to a TransformElement.
|
||||
/// </summary>
|
||||
public abstract TransformerModel ToModel();
|
||||
public abstract TransformElement ToModel();
|
||||
|
||||
/// <summary>
|
||||
/// Helper to create a JsonElement from an object.
|
||||
/// </summary>
|
||||
protected static JsonElement CreateConfigElement(object config)
|
||||
{
|
||||
var json = JsonSerializer.Serialize(config, JsonOptions);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
return doc.RootElement.Clone();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -35,9 +51,18 @@ public class ColumnDropTransformerViewModel : TransformerStepViewModelBase
|
||||
{
|
||||
private string _columnsText;
|
||||
|
||||
public ColumnDropTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged)
|
||||
public ColumnDropTransformerViewModel(TransformElement element, Action onChanged) : base(onChanged)
|
||||
{
|
||||
_columnsText = model.Columns != null ? string.Join("\n", model.Columns) : string.Empty;
|
||||
_columnsText = string.Empty;
|
||||
if (element.Config.HasValue)
|
||||
{
|
||||
if (element.Config.Value.TryGetProperty("columns", out var columnsProp) &&
|
||||
columnsProp.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
var columns = columnsProp.EnumerateArray().Select(c => c.GetString() ?? "").Where(c => !string.IsNullOrEmpty(c));
|
||||
_columnsText = string.Join("\n", columns);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ColumnDropTransformerViewModel(Action onChanged) : base(onChanged)
|
||||
@@ -78,10 +103,10 @@ public class ColumnDropTransformerViewModel : TransformerStepViewModelBase
|
||||
return _columnsText.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Length;
|
||||
}
|
||||
|
||||
public override TransformerModel ToModel() => new()
|
||||
public override TransformElement ToModel() => new()
|
||||
{
|
||||
Type = TransformerType,
|
||||
Columns = GetColumns()
|
||||
TransformType = TransformerType,
|
||||
Config = CreateConfigElement(new { columns = GetColumns() })
|
||||
};
|
||||
}
|
||||
|
||||
@@ -90,14 +115,25 @@ public class ColumnDropTransformerViewModel : TransformerStepViewModelBase
|
||||
/// </summary>
|
||||
public class ColumnRenameTransformerViewModel : TransformerStepViewModelBase
|
||||
{
|
||||
public ColumnRenameTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged)
|
||||
public ColumnRenameTransformerViewModel(TransformElement element, Action onChanged) : base(onChanged)
|
||||
{
|
||||
Mappings = new ObservableCollection<ColumnMappingViewModel>(
|
||||
model.Mappings?.Select(kvp => new ColumnMappingViewModel(kvp.Key, kvp.Value, () =>
|
||||
Mappings = [];
|
||||
if (element.Config.HasValue)
|
||||
{
|
||||
if (element.Config.Value.TryGetProperty("mappings", out var mappingsProp) &&
|
||||
mappingsProp.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
OnPropertyChanged(nameof(Summary));
|
||||
NotifyChanged();
|
||||
})) ?? []);
|
||||
foreach (var prop in mappingsProp.EnumerateObject())
|
||||
{
|
||||
var newName = prop.Value.GetString() ?? "";
|
||||
Mappings.Add(new ColumnMappingViewModel(prop.Name, newName, () =>
|
||||
{
|
||||
OnPropertyChanged(nameof(Summary));
|
||||
NotifyChanged();
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
AddMappingCommand = new RelayCommand(AddMapping);
|
||||
}
|
||||
|
||||
@@ -147,10 +183,10 @@ public class ColumnRenameTransformerViewModel : TransformerStepViewModelBase
|
||||
}
|
||||
}
|
||||
|
||||
public override TransformerModel ToModel() => new()
|
||||
public override TransformElement ToModel() => new()
|
||||
{
|
||||
Type = TransformerType,
|
||||
Mappings = Mappings.ToDictionary(m => m.OldName, m => m.NewName)
|
||||
TransformType = TransformerType,
|
||||
Config = CreateConfigElement(new { mappings = Mappings.ToDictionary(m => m.OldName, m => m.NewName) })
|
||||
};
|
||||
}
|
||||
|
||||
@@ -206,11 +242,21 @@ public class JdeDateTransformerViewModel : TransformerStepViewModelBase
|
||||
private string? _timeColumn;
|
||||
private string? _outputColumn;
|
||||
|
||||
public JdeDateTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged)
|
||||
public JdeDateTransformerViewModel(TransformElement element, Action onChanged) : base(onChanged)
|
||||
{
|
||||
_dateColumn = model.DateColumn;
|
||||
_timeColumn = model.TimeColumn;
|
||||
_outputColumn = model.OutputColumn;
|
||||
_dateColumn = null;
|
||||
_timeColumn = null;
|
||||
_outputColumn = null;
|
||||
|
||||
if (element.Config.HasValue)
|
||||
{
|
||||
if (element.Config.Value.TryGetProperty("dateColumn", out var dateProp))
|
||||
_dateColumn = dateProp.GetString();
|
||||
if (element.Config.Value.TryGetProperty("timeColumn", out var timeProp))
|
||||
_timeColumn = timeProp.GetString();
|
||||
if (element.Config.Value.TryGetProperty("outputColumn", out var outputProp))
|
||||
_outputColumn = outputProp.GetString();
|
||||
}
|
||||
}
|
||||
|
||||
public JdeDateTransformerViewModel(Action onChanged) : base(onChanged)
|
||||
@@ -270,15 +316,26 @@ public class JdeDateTransformerViewModel : TransformerStepViewModelBase
|
||||
}
|
||||
}
|
||||
|
||||
public override TransformerModel ToModel() => new()
|
||||
public override TransformElement ToModel() => new()
|
||||
{
|
||||
Type = TransformerType,
|
||||
DateColumn = _dateColumn,
|
||||
TimeColumn = _timeColumn,
|
||||
OutputColumn = _outputColumn
|
||||
TransformType = TransformerType,
|
||||
Config = CreateConfigElement(new { dateColumn = _dateColumn, timeColumn = _timeColumn, outputColumn = _outputColumn })
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Specifies behavior when a regex pattern does not match the input value.
|
||||
/// </summary>
|
||||
public enum NonMatchBehavior
|
||||
{
|
||||
/// <summary>Keep the original value unchanged.</summary>
|
||||
KeepOriginal,
|
||||
/// <summary>Return null/DBNull.</summary>
|
||||
ReturnNull,
|
||||
/// <summary>Return an empty string.</summary>
|
||||
ReturnEmpty
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// View model for Regex transformer.
|
||||
/// </summary>
|
||||
@@ -301,14 +358,41 @@ public class RegexTransformerViewModel : TransformerStepViewModelBase
|
||||
private bool _hasTestError;
|
||||
private string _testErrorMessage = string.Empty;
|
||||
|
||||
public RegexTransformerViewModel(TransformerModel model, Action onChanged) : base(onChanged)
|
||||
public RegexTransformerViewModel(TransformElement element, Action onChanged) : base(onChanged)
|
||||
{
|
||||
_columnName = model.ColumnName ?? string.Empty;
|
||||
_pattern = model.Pattern ?? string.Empty;
|
||||
_replacement = model.Replacement;
|
||||
_isFindReplaceMode = model.Replacement != null;
|
||||
_ignoreCase = model.IgnoreCase;
|
||||
_nonMatchBehavior = model.NonMatchBehavior;
|
||||
_columnName = string.Empty;
|
||||
_pattern = string.Empty;
|
||||
_replacement = null;
|
||||
_isFindReplaceMode = true;
|
||||
_ignoreCase = false;
|
||||
_nonMatchBehavior = NonMatchBehavior.KeepOriginal;
|
||||
|
||||
if (element.Config.HasValue)
|
||||
{
|
||||
if (element.Config.Value.TryGetProperty("columnName", out var colProp))
|
||||
_columnName = colProp.GetString() ?? string.Empty;
|
||||
if (element.Config.Value.TryGetProperty("pattern", out var patternProp))
|
||||
_pattern = patternProp.GetString() ?? string.Empty;
|
||||
if (element.Config.Value.TryGetProperty("replacement", out var replaceProp))
|
||||
{
|
||||
_replacement = replaceProp.ValueKind == JsonValueKind.Null ? null : replaceProp.GetString();
|
||||
_isFindReplaceMode = _replacement != null;
|
||||
}
|
||||
else
|
||||
{
|
||||
// No replacement property means Match & Extract mode
|
||||
_replacement = null;
|
||||
_isFindReplaceMode = false;
|
||||
}
|
||||
if (element.Config.Value.TryGetProperty("ignoreCase", out var ignoreProp))
|
||||
_ignoreCase = ignoreProp.GetBoolean();
|
||||
if (element.Config.Value.TryGetProperty("nonMatchBehavior", out var behaviorProp))
|
||||
{
|
||||
var behaviorStr = behaviorProp.GetString();
|
||||
if (Enum.TryParse<NonMatchBehavior>(behaviorStr, true, out var behavior))
|
||||
_nonMatchBehavior = behavior;
|
||||
}
|
||||
}
|
||||
TestPatternCommand = new RelayCommand(ExecuteTestPattern);
|
||||
}
|
||||
|
||||
@@ -544,14 +628,17 @@ public class RegexTransformerViewModel : TransformerStepViewModelBase
|
||||
TestErrorMessage = string.Empty;
|
||||
}
|
||||
|
||||
public override TransformerModel ToModel() => new()
|
||||
public override TransformElement ToModel() => new()
|
||||
{
|
||||
Type = TransformerType,
|
||||
ColumnName = _columnName,
|
||||
Pattern = _pattern,
|
||||
Replacement = _isFindReplaceMode ? _replacement : null,
|
||||
IgnoreCase = _ignoreCase,
|
||||
NonMatchBehavior = _nonMatchBehavior
|
||||
TransformType = TransformerType,
|
||||
Config = CreateConfigElement(new
|
||||
{
|
||||
columnName = _columnName,
|
||||
pattern = _pattern,
|
||||
replacement = _isFindReplaceMode ? _replacement : null,
|
||||
ignoreCase = _ignoreCase,
|
||||
nonMatchBehavior = _nonMatchBehavior.ToString()
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -561,16 +648,16 @@ public class RegexTransformerViewModel : TransformerStepViewModelBase
|
||||
public static class TransformerFactory
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a transformer view model from a model.
|
||||
/// Creates a transformer view model from a TransformElement.
|
||||
/// </summary>
|
||||
public static TransformerStepViewModelBase? Create(TransformerModel model, Action onChanged)
|
||||
public static TransformerStepViewModelBase? Create(TransformElement element, Action onChanged)
|
||||
{
|
||||
return model.Type?.ToLowerInvariant() switch
|
||||
return element.TransformType?.ToLowerInvariant() switch
|
||||
{
|
||||
"columndrop" => new ColumnDropTransformerViewModel(model, onChanged),
|
||||
"columnrename" => new ColumnRenameTransformerViewModel(model, onChanged),
|
||||
"jdedate" => new JdeDateTransformerViewModel(model, onChanged),
|
||||
"regex" => new RegexTransformerViewModel(model, onChanged),
|
||||
"columndrop" => new ColumnDropTransformerViewModel(element, onChanged),
|
||||
"columnrename" => new ColumnRenameTransformerViewModel(element, onChanged),
|
||||
"jdedate" => new JdeDateTransformerViewModel(element, onChanged),
|
||||
"regex" => new RegexTransformerViewModel(element, onChanged),
|
||||
_ => null // Unknown transformer type
|
||||
};
|
||||
}
|
||||
|
||||
@@ -13,25 +13,6 @@
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Destination Type Toggle -->
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Load Type" Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<StackPanel Orientation="Horizontal" Spacing="12">
|
||||
<RadioButton GroupName="DestType"
|
||||
IsChecked="{Binding IsBulkMerge}"
|
||||
Foreground="#E6EDF5">
|
||||
<TextBlock Text="Bulk Merge (Upsert)" FontSize="12"/>
|
||||
</RadioButton>
|
||||
<RadioButton GroupName="DestType"
|
||||
IsChecked="{Binding !IsBulkMerge}"
|
||||
Foreground="#E6EDF5">
|
||||
<TextBlock Text="Bulk Import (Truncate+Load)" FontSize="12"/>
|
||||
</RadioButton>
|
||||
</StackPanel>
|
||||
<TextBlock Text="{Binding TypeDescription}"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Destination Table -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
@@ -48,55 +29,39 @@
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- BulkMerge-specific fields -->
|
||||
<StackPanel Spacing="16" IsVisible="{Binding IsBulkMerge}">
|
||||
<!-- Match Columns -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Match Columns (one per line)"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding MatchColumnsText}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="11"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="80"
|
||||
Watermark="OrderNumber
OrderType"/>
|
||||
<TextBlock Text="Columns to match source rows with existing destination rows"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Exclude From Update -->
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Exclude From Update (one per line)"
|
||||
<!-- Match Columns -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Match Columns (one per line)"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBox Text="{Binding ExcludeFromUpdateText}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="11"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="60"
|
||||
Watermark="CreatedDate
CreatedBy"/>
|
||||
<TextBlock Text="Columns to skip when updating existing rows"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding MatchColumnsText}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="11"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="80"
|
||||
Watermark="OrderNumber
OrderType"/>
|
||||
<TextBlock Text="Columns to match source rows with existing destination rows (for upsert)"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- BulkImport info box -->
|
||||
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
|
||||
CornerRadius="4" Padding="12" Margin="0,8,0,0"
|
||||
IsVisible="{Binding !IsBulkMerge}">
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Bulk Import Mode" Foreground="#F59E0B" FontSize="11" FontWeight="Medium"/>
|
||||
<TextBlock Text="All existing data will be deleted before loading new data."
|
||||
Foreground="#5C6A7A" FontSize="10"/>
|
||||
<TextBlock Text="Use this for full table refreshes during mass sync operations."
|
||||
Foreground="#5C6A7A" FontSize="10"/>
|
||||
</StackPanel>
|
||||
</Border>
|
||||
<!-- Exclude From Update -->
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Exclude From Update (one per line)"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBox Text="{Binding ExcludeFromUpdateText}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="11"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="60"
|
||||
Watermark="CreatedDate
CreatedBy"/>
|
||||
<TextBlock Text="Columns to skip when updating existing rows"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
</StackPanel>
|
||||
</UserControl>
|
||||
|
||||
@@ -13,145 +13,106 @@
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Source Type Toggle -->
|
||||
<!-- Connection -->
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Source Type" Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<StackPanel Orientation="Horizontal" Spacing="12">
|
||||
<RadioButton GroupName="SourceType"
|
||||
IsChecked="{Binding IsDatabaseSource}"
|
||||
Foreground="#E6EDF5">
|
||||
<TextBlock Text="Database" FontSize="12"/>
|
||||
</RadioButton>
|
||||
<RadioButton GroupName="SourceType"
|
||||
IsChecked="{Binding IsFileSource}"
|
||||
Foreground="#E6EDF5">
|
||||
<TextBlock Text="File" FontSize="12"/>
|
||||
</RadioButton>
|
||||
</StackPanel>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Database Source Fields -->
|
||||
<StackPanel Spacing="16" IsVisible="{Binding IsDatabaseSource}">
|
||||
<!-- Connection -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Connection"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<ComboBox ItemsSource="{Binding AvailableConnections}"
|
||||
SelectedItem="{Binding Connection}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="36"
|
||||
HorizontalAlignment="Stretch"
|
||||
PlaceholderText="Select connection..."/>
|
||||
<TextBlock Text="Connection string name from Settings > ConnectionStrings"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Query -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Query"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding Query}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="11"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="100"
|
||||
Watermark="SELECT ... FROM ... WHERE ..."/>
|
||||
<TextBlock Text="SQL query for incremental updates (use @LastSync parameter)"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Mass Query -->
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Mass Query (Optional)"
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Connection"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBox Text="{Binding MassQuery}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="11"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="80"
|
||||
Watermark="SELECT ... FROM ... (no date filter)"/>
|
||||
<TextBlock Text="Query for full table reload during mass sync"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Parameters Section -->
|
||||
<Expander IsExpanded="False">
|
||||
<Expander.Header>
|
||||
<StackPanel Orientation="Horizontal" Spacing="8">
|
||||
<TextBlock Text="Parameters" Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="{Binding Parameters.Count, StringFormat='({0})'}"
|
||||
Foreground="#5C6A7A" FontSize="12"/>
|
||||
</StackPanel>
|
||||
</Expander.Header>
|
||||
<StackPanel Spacing="8" Margin="0,8,0,0">
|
||||
<ItemsControl ItemsSource="{Binding Parameters}">
|
||||
<ItemsControl.ItemTemplate>
|
||||
<DataTemplate DataType="steps:ParameterViewModel">
|
||||
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
|
||||
CornerRadius="4" Padding="8" Margin="0,0,0,4">
|
||||
<Grid ColumnDefinitions="*,8,*,8,*,8,Auto">
|
||||
<StackPanel Grid.Column="0" Spacing="2">
|
||||
<TextBlock Text="Key" Foreground="#5C6A7A" FontSize="10"/>
|
||||
<TextBox Text="{Binding Key}"
|
||||
Background="#232A35" Height="28" FontSize="11"/>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="2" Spacing="2">
|
||||
<TextBlock Text="Format" Foreground="#5C6A7A" FontSize="10"/>
|
||||
<TextBox Text="{Binding Format}"
|
||||
Background="#232A35" Height="28" FontSize="11"
|
||||
Watermark="jdeJulian"/>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="4" Spacing="2">
|
||||
<TextBlock Text="Source" Foreground="#5C6A7A" FontSize="10"/>
|
||||
<TextBox Text="{Binding Source}"
|
||||
Background="#232A35" Height="28" FontSize="11"
|
||||
Watermark="offset"/>
|
||||
</StackPanel>
|
||||
<Button Grid.Column="6" Content="X"
|
||||
Background="Transparent" Foreground="#FF6B6B"
|
||||
BorderThickness="0" FontSize="11" Width="24"
|
||||
VerticalAlignment="Bottom"/>
|
||||
</Grid>
|
||||
</Border>
|
||||
</DataTemplate>
|
||||
</ItemsControl.ItemTemplate>
|
||||
</ItemsControl>
|
||||
<Button Content="+ Add Parameter"
|
||||
Background="#232A35" Foreground="#9BA8B8"
|
||||
BorderBrush="#3D4550" Height="32"
|
||||
HorizontalAlignment="Left" Padding="12,0"
|
||||
Command="{Binding AddParameterCommand}"/>
|
||||
</StackPanel>
|
||||
</Expander>
|
||||
<ComboBox ItemsSource="{Binding AvailableConnections}"
|
||||
SelectedItem="{Binding Connection}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="36"
|
||||
HorizontalAlignment="Stretch"
|
||||
PlaceholderText="Select connection..."/>
|
||||
<TextBlock Text="Connection string name from Settings > ConnectionStrings"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- File Source Fields -->
|
||||
<StackPanel Spacing="16" IsVisible="{Binding IsFileSource}">
|
||||
<!-- File Name -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="File Name"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding FileName}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="36"
|
||||
FontFamily="JetBrains Mono"
|
||||
Watermark="data.pb.zstd"/>
|
||||
<TextBlock Text="Protobuf+Zstd compressed file name"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
<!-- Query -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Query"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding Query}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="11"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="100"
|
||||
Watermark="SELECT ... FROM ... WHERE ..."/>
|
||||
<TextBlock Text="SQL query for incremental updates (use @LastSync parameter)"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Mass Query -->
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Mass Query (Optional)"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBox Text="{Binding MassQuery}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="11"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="80"
|
||||
Watermark="SELECT ... FROM ... (no date filter)"/>
|
||||
<TextBlock Text="Query for full table reload during mass sync"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Parameters Section -->
|
||||
<Expander IsExpanded="False">
|
||||
<Expander.Header>
|
||||
<StackPanel Orientation="Horizontal" Spacing="8">
|
||||
<TextBlock Text="Parameters" Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="{Binding Parameters.Count, StringFormat='({0})'}"
|
||||
Foreground="#5C6A7A" FontSize="12"/>
|
||||
</StackPanel>
|
||||
</Expander.Header>
|
||||
<StackPanel Spacing="8" Margin="0,8,0,0">
|
||||
<ItemsControl ItemsSource="{Binding Parameters}">
|
||||
<ItemsControl.ItemTemplate>
|
||||
<DataTemplate DataType="steps:ParameterViewModel">
|
||||
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
|
||||
CornerRadius="4" Padding="8" Margin="0,0,0,4">
|
||||
<Grid ColumnDefinitions="*,8,*,8,*,8,Auto">
|
||||
<StackPanel Grid.Column="0" Spacing="2">
|
||||
<TextBlock Text="Key" Foreground="#5C6A7A" FontSize="10"/>
|
||||
<TextBox Text="{Binding Key}"
|
||||
Background="#232A35" Height="28" FontSize="11"/>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="2" Spacing="2">
|
||||
<TextBlock Text="Format" Foreground="#5C6A7A" FontSize="10"/>
|
||||
<TextBox Text="{Binding Format}"
|
||||
Background="#232A35" Height="28" FontSize="11"
|
||||
Watermark="jdeJulian"/>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="4" Spacing="2">
|
||||
<TextBlock Text="Source" Foreground="#5C6A7A" FontSize="10"/>
|
||||
<TextBox Text="{Binding Source}"
|
||||
Background="#232A35" Height="28" FontSize="11"
|
||||
Watermark="offset"/>
|
||||
</StackPanel>
|
||||
<Button Grid.Column="6" Content="X"
|
||||
Background="Transparent" Foreground="#FF6B6B"
|
||||
BorderThickness="0" FontSize="11" Width="24"
|
||||
VerticalAlignment="Bottom"/>
|
||||
</Grid>
|
||||
</Border>
|
||||
</DataTemplate>
|
||||
</ItemsControl.ItemTemplate>
|
||||
</ItemsControl>
|
||||
<Button Content="+ Add Parameter"
|
||||
Background="#232A35" Foreground="#9BA8B8"
|
||||
BorderBrush="#3D4550" Height="32"
|
||||
HorizontalAlignment="Left" Padding="12,0"
|
||||
Command="{Binding AddParameterCommand}"/>
|
||||
</StackPanel>
|
||||
</Expander>
|
||||
</StackPanel>
|
||||
</UserControl>
|
||||
|
||||
@@ -90,30 +90,30 @@
|
||||
</Expander.Header>
|
||||
<StackPanel Spacing="8" Margin="0,8,0,0">
|
||||
<StackPanel Orientation="Horizontal" Spacing="8">
|
||||
<CheckBox IsChecked="{Binding MassSchedule.Enabled}"/>
|
||||
<CheckBox IsChecked="{Binding MassSyncEnabled}"/>
|
||||
<TextBlock Text="Mass" Foreground="#9BA8B8" FontSize="12"/>
|
||||
<NumericUpDown Value="{Binding MassSchedule.IntervalMinutes}"
|
||||
<NumericUpDown Value="{Binding MassSyncIntervalMinutes}"
|
||||
Minimum="1" Width="80" Height="28"
|
||||
Background="#232A35" FontSize="11"
|
||||
IsEnabled="{Binding MassSchedule.Enabled}"/>
|
||||
IsEnabled="{Binding MassSyncEnabled}"/>
|
||||
<TextBlock Text="min" Foreground="#5C6A7A" FontSize="11" VerticalAlignment="Center"/>
|
||||
</StackPanel>
|
||||
<StackPanel Orientation="Horizontal" Spacing="8">
|
||||
<CheckBox IsChecked="{Binding DailySchedule.Enabled}"/>
|
||||
<CheckBox IsChecked="{Binding DailySyncEnabled}"/>
|
||||
<TextBlock Text="Daily" Foreground="#9BA8B8" FontSize="12"/>
|
||||
<NumericUpDown Value="{Binding DailySchedule.IntervalMinutes}"
|
||||
<NumericUpDown Value="{Binding DailySyncIntervalMinutes}"
|
||||
Minimum="1" Width="80" Height="28"
|
||||
Background="#232A35" FontSize="11"
|
||||
IsEnabled="{Binding DailySchedule.Enabled}"/>
|
||||
IsEnabled="{Binding DailySyncEnabled}"/>
|
||||
<TextBlock Text="min" Foreground="#5C6A7A" FontSize="11" VerticalAlignment="Center"/>
|
||||
</StackPanel>
|
||||
<StackPanel Orientation="Horizontal" Spacing="8">
|
||||
<CheckBox IsChecked="{Binding HourlySchedule.Enabled}"/>
|
||||
<CheckBox IsChecked="{Binding HourlySyncEnabled}"/>
|
||||
<TextBlock Text="Hourly" Foreground="#9BA8B8" FontSize="12"/>
|
||||
<NumericUpDown Value="{Binding HourlySchedule.IntervalMinutes}"
|
||||
<NumericUpDown Value="{Binding HourlySyncIntervalMinutes}"
|
||||
Minimum="1" Width="80" Height="28"
|
||||
Background="#232A35" FontSize="11"
|
||||
IsEnabled="{Binding HourlySchedule.Enabled}"/>
|
||||
IsEnabled="{Binding HourlySyncEnabled}"/>
|
||||
<TextBlock Text="min" Foreground="#5C6A7A" FontSize="11" VerticalAlignment="Center"/>
|
||||
</StackPanel>
|
||||
</StackPanel>
|
||||
|
||||
@@ -1,261 +0,0 @@
|
||||
<UserControl xmlns="https://github.com/avaloniaui"
|
||||
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||
xmlns:vm="using:JdeScoping.ConfigManager.ViewModels.Forms"
|
||||
x:Class="JdeScoping.ConfigManager.Views.Forms.PipelineFormView"
|
||||
x:DataType="vm:PipelineFormViewModel">
|
||||
|
||||
<ScrollViewer VerticalScrollBarVisibility="Auto">
|
||||
<StackPanel Spacing="16" MaxWidth="700">
|
||||
<!-- Header -->
|
||||
<StackPanel>
|
||||
<TextBlock Text="{Binding Name, StringFormat='{}{0} Pipeline'}"
|
||||
Foreground="#E6EDF5" FontSize="18" FontWeight="SemiBold"/>
|
||||
<Border Height="1" Background="#2D3540" Margin="0,12,0,0"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Source Section (Expander) -->
|
||||
<Expander IsExpanded="True">
|
||||
<Expander.Header>
|
||||
<TextBlock Text="Source" Foreground="#E6EDF5" FontWeight="SemiBold" FontSize="14"/>
|
||||
</Expander.Header>
|
||||
<Border Background="#0D0F12" BorderBrush="#2D3540" BorderThickness="1,0,1,1"
|
||||
CornerRadius="0,0,6,6" Padding="16">
|
||||
<StackPanel Spacing="16">
|
||||
<!-- Connection -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Connection"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding Connection}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="36"
|
||||
FontFamily="JetBrains Mono"
|
||||
Watermark="JdeOracle"/>
|
||||
<TextBlock Text="Name of the connection string to use"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Query -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Query"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding Query}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="12"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="120"
|
||||
Watermark="SELECT ... FROM ... WHERE ..."/>
|
||||
<TextBlock Text="SQL query for incremental updates (use @LastSync parameter)"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Mass Query -->
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Mass Query (Optional)"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBox Text="{Binding MassQuery}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="12"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="80"
|
||||
Watermark="SELECT ... FROM ... (no date filter)"/>
|
||||
<TextBlock Text="Alternative query for mass sync (full table reload)"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
</StackPanel>
|
||||
</Border>
|
||||
</Expander>
|
||||
|
||||
<!-- Schedules Section (Expander) -->
|
||||
<Expander IsExpanded="False">
|
||||
<Expander.Header>
|
||||
<TextBlock Text="Schedules" Foreground="#E6EDF5" FontWeight="SemiBold" FontSize="14"/>
|
||||
</Expander.Header>
|
||||
<Border Background="#0D0F12" BorderBrush="#2D3540" BorderThickness="1,0,1,1"
|
||||
CornerRadius="0,0,6,6" Padding="16">
|
||||
<StackPanel Spacing="20">
|
||||
<!-- Mass Schedule -->
|
||||
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
|
||||
CornerRadius="4" Padding="12">
|
||||
<StackPanel Spacing="12">
|
||||
<StackPanel Orientation="Horizontal" Spacing="8">
|
||||
<CheckBox IsChecked="{Binding MassSchedule.Enabled}"/>
|
||||
<TextBlock Text="Mass Schedule" Foreground="#E6EDF5" FontWeight="Medium"/>
|
||||
</StackPanel>
|
||||
<Grid ColumnDefinitions="*,12,*,12,*" IsEnabled="{Binding MassSchedule.Enabled}">
|
||||
<StackPanel Grid.Column="0" Spacing="4">
|
||||
<TextBlock Text="Interval (min)" Foreground="#9BA8B8" FontSize="11"/>
|
||||
<NumericUpDown Value="{Binding MassSchedule.IntervalMinutes}"
|
||||
Minimum="1" Maximum="10080"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="32"
|
||||
FontFamily="JetBrains Mono" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="2" Spacing="4">
|
||||
<CheckBox IsChecked="{Binding MassSchedule.PrePurge}" Foreground="#9BA8B8">
|
||||
<TextBlock Text="Pre-Purge" Foreground="#9BA8B8" FontSize="11"/>
|
||||
</CheckBox>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="4" Spacing="4">
|
||||
<CheckBox IsChecked="{Binding MassSchedule.ReIndex}" Foreground="#9BA8B8">
|
||||
<TextBlock Text="Re-Index" Foreground="#9BA8B8" FontSize="11"/>
|
||||
</CheckBox>
|
||||
</StackPanel>
|
||||
</Grid>
|
||||
</StackPanel>
|
||||
</Border>
|
||||
|
||||
<!-- Daily Schedule -->
|
||||
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
|
||||
CornerRadius="4" Padding="12">
|
||||
<StackPanel Spacing="12">
|
||||
<StackPanel Orientation="Horizontal" Spacing="8">
|
||||
<CheckBox IsChecked="{Binding DailySchedule.Enabled}"/>
|
||||
<TextBlock Text="Daily Schedule" Foreground="#E6EDF5" FontWeight="Medium"/>
|
||||
</StackPanel>
|
||||
<Grid ColumnDefinitions="*,12,*,12,*" IsEnabled="{Binding DailySchedule.Enabled}">
|
||||
<StackPanel Grid.Column="0" Spacing="4">
|
||||
<TextBlock Text="Interval (min)" Foreground="#9BA8B8" FontSize="11"/>
|
||||
<NumericUpDown Value="{Binding DailySchedule.IntervalMinutes}"
|
||||
Minimum="1" Maximum="1440"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="32"
|
||||
FontFamily="JetBrains Mono" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="2" Spacing="4">
|
||||
<CheckBox IsChecked="{Binding DailySchedule.PrePurge}" Foreground="#9BA8B8">
|
||||
<TextBlock Text="Pre-Purge" Foreground="#9BA8B8" FontSize="11"/>
|
||||
</CheckBox>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="4" Spacing="4">
|
||||
<CheckBox IsChecked="{Binding DailySchedule.ReIndex}" Foreground="#9BA8B8">
|
||||
<TextBlock Text="Re-Index" Foreground="#9BA8B8" FontSize="11"/>
|
||||
</CheckBox>
|
||||
</StackPanel>
|
||||
</Grid>
|
||||
</StackPanel>
|
||||
</Border>
|
||||
|
||||
<!-- Hourly Schedule -->
|
||||
<Border Background="#151920" BorderBrush="#2D3540" BorderThickness="1"
|
||||
CornerRadius="4" Padding="12">
|
||||
<StackPanel Spacing="12">
|
||||
<StackPanel Orientation="Horizontal" Spacing="8">
|
||||
<CheckBox IsChecked="{Binding HourlySchedule.Enabled}"/>
|
||||
<TextBlock Text="Hourly Schedule" Foreground="#E6EDF5" FontWeight="Medium"/>
|
||||
</StackPanel>
|
||||
<Grid ColumnDefinitions="*,12,*,12,*" IsEnabled="{Binding HourlySchedule.Enabled}">
|
||||
<StackPanel Grid.Column="0" Spacing="4">
|
||||
<TextBlock Text="Interval (min)" Foreground="#9BA8B8" FontSize="11"/>
|
||||
<NumericUpDown Value="{Binding HourlySchedule.IntervalMinutes}"
|
||||
Minimum="1" Maximum="60"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="32"
|
||||
FontFamily="JetBrains Mono" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="2" Spacing="4">
|
||||
<CheckBox IsChecked="{Binding HourlySchedule.PrePurge}" Foreground="#9BA8B8">
|
||||
<TextBlock Text="Pre-Purge" Foreground="#9BA8B8" FontSize="11"/>
|
||||
</CheckBox>
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="4" Spacing="4">
|
||||
<CheckBox IsChecked="{Binding HourlySchedule.ReIndex}" Foreground="#9BA8B8">
|
||||
<TextBlock Text="Re-Index" Foreground="#9BA8B8" FontSize="11"/>
|
||||
</CheckBox>
|
||||
</StackPanel>
|
||||
</Grid>
|
||||
</StackPanel>
|
||||
</Border>
|
||||
</StackPanel>
|
||||
</Border>
|
||||
</Expander>
|
||||
|
||||
<!-- Destination Section (Expander) -->
|
||||
<Expander IsExpanded="False">
|
||||
<Expander.Header>
|
||||
<TextBlock Text="Destination" Foreground="#E6EDF5" FontWeight="SemiBold" FontSize="14"/>
|
||||
</Expander.Header>
|
||||
<Border Background="#0D0F12" BorderBrush="#2D3540" BorderThickness="1,0,1,1"
|
||||
CornerRadius="0,0,6,6" Padding="16">
|
||||
<StackPanel Spacing="16">
|
||||
<!-- Destination Table -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Destination Table"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding DestinationTable}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550" Height="36"
|
||||
FontFamily="JetBrains Mono"
|
||||
Watermark="dbo.WorkOrder_Curr"/>
|
||||
<TextBlock Text="Target table in SQL Server (include schema)"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Match Columns -->
|
||||
<StackPanel Spacing="4">
|
||||
<StackPanel Orientation="Horizontal" Spacing="2">
|
||||
<TextBlock Text="Match Columns (one per line)"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBlock Text="*" Foreground="#FF6B6B" FontSize="12"/>
|
||||
</StackPanel>
|
||||
<TextBox Text="{Binding MatchColumnsText}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="12"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="60"
|
||||
Watermark="OrderNumber
OrderType"/>
|
||||
<TextBlock Text="Columns used to match source rows to existing destination rows"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Exclude From Update -->
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Exclude From Update (one per line)"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBox Text="{Binding ExcludeFromUpdateText}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="12"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="60"
|
||||
Watermark="CreatedDate
CreatedBy"/>
|
||||
<TextBlock Text="Columns to skip when updating existing rows"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Post Scripts -->
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Text="Post Scripts (one per line)"
|
||||
Foreground="#9BA8B8" FontSize="12" FontWeight="Medium"/>
|
||||
<TextBox Text="{Binding PostScriptsText}"
|
||||
Background="#232A35" Foreground="#E6EDF5"
|
||||
BorderBrush="#3D4550"
|
||||
FontFamily="JetBrains Mono" FontSize="12"
|
||||
AcceptsReturn="True"
|
||||
TextWrapping="NoWrap"
|
||||
MinHeight="60"
|
||||
Watermark="EXEC dbo.UpdateStats @TableName"/>
|
||||
<TextBlock Text="SQL scripts to execute after sync completes"
|
||||
Foreground="#5C6A7A" FontSize="11"/>
|
||||
</StackPanel>
|
||||
</StackPanel>
|
||||
</Border>
|
||||
</Expander>
|
||||
</StackPanel>
|
||||
</ScrollViewer>
|
||||
</UserControl>
|
||||
@@ -1,11 +0,0 @@
|
||||
using Avalonia.Controls;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Views.Forms;
|
||||
|
||||
public partial class PipelineFormView : UserControl
|
||||
{
|
||||
public PipelineFormView()
|
||||
{
|
||||
InitializeComponent();
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.ConfigManager.Services;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Tests.Services;
|
||||
|
||||
@@ -47,19 +48,16 @@ public class ValidationServiceTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipelines_WithDuplicateNames_ReturnsError()
|
||||
public void ValidatePipelines_WithEmptyName_ReturnsError()
|
||||
{
|
||||
// Arrange - duplicate keys not possible in dictionary, but empty names are invalid
|
||||
var config = new PipelinesConfigModel
|
||||
// Arrange
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
{
|
||||
[""] = new PipelineModel()
|
||||
}
|
||||
[""] = new EtlPipelineConfig()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipelines(config);
|
||||
var result = _sut.ValidatePipelines(pipelines);
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
@@ -69,22 +67,140 @@ public class ValidationServiceTests
|
||||
public void ValidatePipelines_WithInvalidConnection_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesConfigModel
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
["Test"] = new EtlPipelineConfig
|
||||
{
|
||||
["Test"] = new PipelineModel
|
||||
{
|
||||
Source = new PipelineSource { Connection = "invalid" }
|
||||
}
|
||||
Source = new SourceElement { Connection = "invalid" }
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipelines(config);
|
||||
var result = _sut.ValidatePipelines(pipelines);
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Connection"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithValidConfig_ReturnsNoErrors()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "Test",
|
||||
IsEnabled = true,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM Test"
|
||||
},
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Table = "TestTable",
|
||||
MatchColumns = ["Id"]
|
||||
},
|
||||
HourlySyncIntervalMinutes = 60
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
result.Errors.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithMissingQuery_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "" },
|
||||
Destination = new DestinationElement { Table = "TestTable" },
|
||||
IsManualOnly = true
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Query"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithNoScheduleAndNotManual_ReturnsWarning()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable", MatchColumns = ["Id"] },
|
||||
IsManualOnly = false // No schedules and not manual-only
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.Warnings.ShouldContain(w => w.Contains("No sync schedule"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_ManualOnly_DoesNotRequireSchedule()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable", MatchColumns = ["Id"] },
|
||||
IsManualOnly = true
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.Warnings.ShouldNotContain(w => w.Contains("No sync schedule"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithIntervalBelowMinimum_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable" },
|
||||
HourlySyncIntervalMinutes = 5 // Below minimum of 15
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Hourly sync interval"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithNoMatchColumns_ReturnsWarning()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable", MatchColumns = [] },
|
||||
IsManualOnly = true
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.Warnings.ShouldContain(w => w.Contains("No MatchColumns"));
|
||||
}
|
||||
}
|
||||
|
||||
-287
@@ -1,287 +0,0 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.ConfigManager.ViewModels.Forms;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Tests.ViewModels.Forms;
|
||||
|
||||
public class PipelineFormViewModelTests
|
||||
{
|
||||
[Fact]
|
||||
public void Constructor_InitializesFromModel()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Source = new PipelineSource
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM Test"
|
||||
},
|
||||
Destination = new PipelineDestination
|
||||
{
|
||||
Table = "TestTable",
|
||||
MatchColumns = ["Id", "Name"]
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var sut = new PipelineFormViewModel("TestPipeline", model, () => { });
|
||||
|
||||
// Assert
|
||||
sut.Name.ShouldBe("TestPipeline");
|
||||
sut.Connection.ShouldBe("jde");
|
||||
sut.Query.ShouldBe("SELECT * FROM Test");
|
||||
sut.DestinationTable.ShouldBe("TestTable");
|
||||
sut.MatchColumnsText.ShouldBe("Id\nName");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PropertyChange_UpdatesModelAndInvokesOnChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel();
|
||||
var changedInvoked = false;
|
||||
var sut = new PipelineFormViewModel("Test", model, () => changedInvoked = true);
|
||||
|
||||
// Act
|
||||
sut.Connection = "cms";
|
||||
|
||||
// Assert
|
||||
model.Source.Connection.ShouldBe("cms");
|
||||
changedInvoked.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MatchColumnsText_SplitsIntoArray()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel();
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Act
|
||||
sut.MatchColumnsText = "Col1\nCol2\nCol3";
|
||||
|
||||
// Assert
|
||||
model.Destination.MatchColumns.Length.ShouldBe(3);
|
||||
model.Destination.MatchColumns[0].ShouldBe("Col1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Schedules_AreInitialized()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Schedules = new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleModel { Enabled = true, IntervalMinutes = 10080 }
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Assert
|
||||
sut.MassSchedule.ShouldNotBeNull();
|
||||
sut.MassSchedule.Enabled.ShouldBeTrue();
|
||||
sut.MassSchedule.IntervalMinutes.ShouldBe(10080);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullSchedules_AreInitializedToDefault()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Schedules = new PipelineSchedules
|
||||
{
|
||||
Mass = null,
|
||||
Daily = null,
|
||||
Hourly = null
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Assert
|
||||
sut.MassSchedule.ShouldNotBeNull();
|
||||
sut.DailySchedule.ShouldNotBeNull();
|
||||
sut.HourlySchedule.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExcludeFromUpdateText_JoinsAndSplits()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Destination = new PipelineDestination
|
||||
{
|
||||
ExcludeFromUpdate = ["CreatedDate", "ModifiedDate"]
|
||||
}
|
||||
};
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Assert - verify getter joins correctly
|
||||
sut.ExcludeFromUpdateText.ShouldBe("CreatedDate\nModifiedDate");
|
||||
|
||||
// Act - verify setter splits correctly
|
||||
sut.ExcludeFromUpdateText = "Col1\nCol2";
|
||||
model.Destination.ExcludeFromUpdate.Length.ShouldBe(2);
|
||||
model.Destination.ExcludeFromUpdate[0].ShouldBe("Col1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PostScriptsText_HandlesNullable()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel { PostScripts = null };
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Assert - null should return empty string
|
||||
sut.PostScriptsText.ShouldBe(string.Empty);
|
||||
|
||||
// Act - set some scripts
|
||||
sut.PostScriptsText = "script1.sql\nscript2.sql";
|
||||
model.PostScripts.ShouldNotBeNull();
|
||||
model.PostScripts!.Length.ShouldBe(2);
|
||||
|
||||
// Act - clear scripts by setting empty
|
||||
sut.PostScriptsText = "";
|
||||
model.PostScripts.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MassQuery_Property_ReadsAndWrites()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Source = new PipelineSource
|
||||
{
|
||||
MassQuery = "SELECT * FROM Test WHERE All = 1"
|
||||
}
|
||||
};
|
||||
var changedInvoked = false;
|
||||
var sut = new PipelineFormViewModel("Test", model, () => changedInvoked = true);
|
||||
|
||||
// Assert - verify getter
|
||||
sut.MassQuery.ShouldBe("SELECT * FROM Test WHERE All = 1");
|
||||
|
||||
// Act - verify setter
|
||||
sut.MassQuery = "SELECT * FROM NewTable";
|
||||
model.Source.MassQuery.ShouldBe("SELECT * FROM NewTable");
|
||||
changedInvoked.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PropertyChange_RaisesPropertyChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel();
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
var propertyChangedRaised = false;
|
||||
sut.PropertyChanged += (s, e) =>
|
||||
{
|
||||
if (e.PropertyName == nameof(PipelineFormViewModel.Query))
|
||||
propertyChangedRaised = true;
|
||||
};
|
||||
|
||||
// Act
|
||||
sut.Query = "SELECT * FROM NewQuery";
|
||||
|
||||
// Assert
|
||||
propertyChangedRaised.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleChange_InvokesOnChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel();
|
||||
var changedInvoked = false;
|
||||
var sut = new PipelineFormViewModel("Test", model, () => changedInvoked = true);
|
||||
|
||||
// Act - change schedule property (Enabled defaults to true, so set to false)
|
||||
sut.MassSchedule.Enabled = false;
|
||||
|
||||
// Assert
|
||||
changedInvoked.ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
|
||||
public class ScheduleFormViewModelTests
|
||||
{
|
||||
[Fact]
|
||||
public void Constructor_InitializesFromModel()
|
||||
{
|
||||
// Arrange
|
||||
var model = new ScheduleModel
|
||||
{
|
||||
Enabled = true,
|
||||
IntervalMinutes = 1440,
|
||||
PrePurge = true,
|
||||
ReIndex = false
|
||||
};
|
||||
|
||||
// Act
|
||||
var sut = new ScheduleFormViewModel(model, () => { });
|
||||
|
||||
// Assert
|
||||
sut.Enabled.ShouldBeTrue();
|
||||
sut.IntervalMinutes.ShouldBe(1440);
|
||||
sut.PrePurge.ShouldBeTrue();
|
||||
sut.ReIndex.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PropertyChange_UpdatesModelAndInvokesOnChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new ScheduleModel();
|
||||
var changedInvoked = false;
|
||||
var sut = new ScheduleFormViewModel(model, () => changedInvoked = true);
|
||||
|
||||
// Act
|
||||
sut.IntervalMinutes = 120;
|
||||
|
||||
// Assert
|
||||
model.IntervalMinutes.ShouldBe(120);
|
||||
changedInvoked.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PropertyChange_RaisesPropertyChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new ScheduleModel();
|
||||
var sut = new ScheduleFormViewModel(model, () => { });
|
||||
var propertyChangedRaised = false;
|
||||
sut.PropertyChanged += (s, e) =>
|
||||
{
|
||||
if (e.PropertyName == nameof(ScheduleFormViewModel.PrePurge))
|
||||
propertyChangedRaised = true;
|
||||
};
|
||||
|
||||
// Act
|
||||
sut.PrePurge = true;
|
||||
|
||||
// Assert
|
||||
propertyChangedRaised.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SameValue_DoesNotInvokeOnChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new ScheduleModel { Enabled = true };
|
||||
var changedInvoked = false;
|
||||
var sut = new ScheduleFormViewModel(model, () => changedInvoked = true);
|
||||
|
||||
// Act - set same value
|
||||
sut.Enabled = true;
|
||||
|
||||
// Assert
|
||||
changedInvoked.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using JdeScoping.ConfigManager.Services;
|
||||
using JdeScoping.ConfigManager.Services.SecureStore;
|
||||
using JdeScoping.ConfigManager.ViewModels;
|
||||
using JdeScoping.ConfigManager.ViewModels.Forms;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NSubstitute;
|
||||
|
||||
@@ -38,7 +39,7 @@ public class MainWindowViewModelTests
|
||||
|
||||
_validationService.ValidateAppSettings(Arg.Any<ConfigModel>())
|
||||
.Returns(new ValidationResult());
|
||||
_validationService.ValidatePipelines(Arg.Any<PipelinesConfigModel>())
|
||||
_validationService.ValidatePipelines(Arg.Any<Dictionary<string, EtlPipelineConfig>>())
|
||||
.Returns(new ValidationResult());
|
||||
}
|
||||
|
||||
@@ -167,15 +168,13 @@ public class MainWindowViewModelTests
|
||||
{
|
||||
// Arrange
|
||||
var config = new ConfigModel();
|
||||
var pipelines = new PipelinesConfigModel
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
["WorkOrders"] = new EtlPipelineConfig
|
||||
{
|
||||
["WorkOrders"] = new PipelineModel
|
||||
{
|
||||
Source = new PipelineSource { Connection = "jde", Query = "SELECT * FROM WO" },
|
||||
Destination = new PipelineDestination { Table = "WorkOrder_Curr" }
|
||||
}
|
||||
Name = "WorkOrders",
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT * FROM WO" },
|
||||
Destination = new DestinationElement { Table = "WorkOrder_Curr" }
|
||||
}
|
||||
};
|
||||
var sut = CreateViewModel();
|
||||
@@ -295,13 +294,10 @@ public class MainWindowViewModelTests
|
||||
{
|
||||
// Arrange
|
||||
var config = new ConfigModel();
|
||||
var pipelines = new PipelinesConfigModel
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
{
|
||||
["Pipeline1"] = new PipelineModel(),
|
||||
["Pipeline2"] = new PipelineModel()
|
||||
}
|
||||
["Pipeline1"] = new EtlPipelineConfig { Name = "Pipeline1" },
|
||||
["Pipeline2"] = new EtlPipelineConfig { Name = "Pipeline2" }
|
||||
};
|
||||
var sut = CreateViewModel();
|
||||
|
||||
|
||||
+50
-28
@@ -1,26 +1,42 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.ConfigManager.ViewModels.PipelineSteps;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Tests.ViewModels;
|
||||
|
||||
public class RegexTransformerViewModelTests
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
private static TransformElement CreateElement(object config)
|
||||
{
|
||||
var json = JsonSerializer.Serialize(config, JsonOptions);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
return new TransformElement
|
||||
{
|
||||
TransformType = "Regex",
|
||||
Config = doc.RootElement.Clone()
|
||||
};
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Constructor_FromModel_LoadsAllProperties()
|
||||
public void Constructor_FromElement_LoadsAllProperties()
|
||||
{
|
||||
// Arrange
|
||||
var model = new TransformerModel
|
||||
var element = CreateElement(new
|
||||
{
|
||||
Type = "Regex",
|
||||
ColumnName = "BatchID",
|
||||
Pattern = "^IIS_",
|
||||
Replacement = "",
|
||||
IgnoreCase = true,
|
||||
NonMatchBehavior = NonMatchBehavior.ReturnEmpty
|
||||
};
|
||||
columnName = "BatchID",
|
||||
pattern = "^IIS_",
|
||||
replacement = "",
|
||||
ignoreCase = true,
|
||||
nonMatchBehavior = "ReturnEmpty"
|
||||
});
|
||||
|
||||
// Act
|
||||
var vm = new RegexTransformerViewModel(model, () => { });
|
||||
var vm = new RegexTransformerViewModel(element, () => { });
|
||||
|
||||
// Assert
|
||||
Assert.Equal("BatchID", vm.ColumnName);
|
||||
@@ -32,19 +48,17 @@ public class RegexTransformerViewModelTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Constructor_FromModel_MatchExtractMode_WhenReplacementNull()
|
||||
public void Constructor_FromElement_MatchExtractMode_WhenReplacementNull()
|
||||
{
|
||||
// Arrange
|
||||
var model = new TransformerModel
|
||||
var element = CreateElement(new
|
||||
{
|
||||
Type = "Regex",
|
||||
ColumnName = "Code",
|
||||
Pattern = @"(\d+)",
|
||||
Replacement = null
|
||||
};
|
||||
columnName = "Code",
|
||||
pattern = @"(\d+)"
|
||||
});
|
||||
|
||||
// Act
|
||||
var vm = new RegexTransformerViewModel(model, () => { });
|
||||
var vm = new RegexTransformerViewModel(element, () => { });
|
||||
|
||||
// Assert
|
||||
Assert.False(vm.IsFindReplaceMode);
|
||||
@@ -66,15 +80,18 @@ public class RegexTransformerViewModelTests
|
||||
};
|
||||
|
||||
// Act
|
||||
var model = vm.ToModel();
|
||||
var element = vm.ToModel();
|
||||
|
||||
// Assert
|
||||
Assert.Equal("Regex", model.Type);
|
||||
Assert.Equal("BatchID", model.ColumnName);
|
||||
Assert.Equal("^IIS_", model.Pattern);
|
||||
Assert.Equal("", model.Replacement);
|
||||
Assert.True(model.IgnoreCase);
|
||||
Assert.Equal(NonMatchBehavior.KeepOriginal, model.NonMatchBehavior);
|
||||
Assert.Equal("Regex", element.TransformType);
|
||||
Assert.True(element.Config.HasValue);
|
||||
|
||||
// Parse the config to verify
|
||||
var config = element.Config!.Value;
|
||||
Assert.Equal("BatchID", config.GetProperty("columnName").GetString());
|
||||
Assert.Equal("^IIS_", config.GetProperty("pattern").GetString());
|
||||
Assert.Equal("", config.GetProperty("replacement").GetString());
|
||||
Assert.True(config.GetProperty("ignoreCase").GetBoolean());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -89,10 +106,15 @@ public class RegexTransformerViewModelTests
|
||||
};
|
||||
|
||||
// Act
|
||||
var model = vm.ToModel();
|
||||
var element = vm.ToModel();
|
||||
|
||||
// Assert
|
||||
Assert.Null(model.Replacement); // null indicates Match & Extract mode
|
||||
Assert.True(element.Config.HasValue);
|
||||
var config = element.Config!.Value;
|
||||
|
||||
// replacement should be null in Match & Extract mode
|
||||
Assert.True(config.TryGetProperty("replacement", out var replacement));
|
||||
Assert.Equal(JsonValueKind.Null, replacement.ValueKind);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Configuration;
|
||||
|
||||
public class PipelinesRootTests
|
||||
{
|
||||
[Fact]
|
||||
public void EffectiveScheduleDefaults_WhenNull_ReturnsDefaults()
|
||||
{
|
||||
var root = new PipelinesRoot(null, null, new Dictionary<string, PipelineConfig>());
|
||||
|
||||
var defaults = root.EffectiveScheduleDefaults;
|
||||
|
||||
defaults.ShouldNotBeNull();
|
||||
defaults.Mass.IntervalMinutes.ShouldBe(10080);
|
||||
defaults.Daily.IntervalMinutes.ShouldBe(1440);
|
||||
defaults.Hourly.IntervalMinutes.ShouldBe(60);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EffectiveScheduleDefaults_WhenProvided_ReturnsProvided()
|
||||
{
|
||||
var customDefaults = new ScheduleDefaults
|
||||
{
|
||||
Mass = new ScheduleConfig { IntervalMinutes = 20000 }
|
||||
};
|
||||
var root = new PipelinesRoot(null, customDefaults, new Dictionary<string, PipelineConfig>());
|
||||
|
||||
var defaults = root.EffectiveScheduleDefaults;
|
||||
|
||||
defaults.Mass.IntervalMinutes.ShouldBe(20000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EffectiveSettings_WhenNull_ReturnsDefaults()
|
||||
{
|
||||
var root = new PipelinesRoot(null, null, new Dictionary<string, PipelineConfig>());
|
||||
|
||||
var settings = root.EffectiveSettings;
|
||||
|
||||
settings.ShouldNotBeNull();
|
||||
settings.Timezone.ShouldBe("UTC");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EffectiveSettings_WhenProvided_ReturnsProvided()
|
||||
{
|
||||
var customSettings = new PipelineSettings("America/New_York");
|
||||
var root = new PipelinesRoot(customSettings, null, new Dictionary<string, PipelineConfig>());
|
||||
|
||||
var settings = root.EffectiveSettings;
|
||||
|
||||
settings.Timezone.ShouldBe("America/New_York");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pipelines_WhenProvided_StoresCorrectly()
|
||||
{
|
||||
var pipelines = new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = CreateMinimalPipelineConfig()
|
||||
};
|
||||
var root = new PipelinesRoot(null, null, pipelines);
|
||||
|
||||
root.Pipelines.ShouldContainKey("TestTable");
|
||||
root.Pipelines["TestTable"].Destination.Table.ShouldBe("TestTable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PipelineConfig_WithSchedules_ParsesCorrectly()
|
||||
{
|
||||
var config = new PipelineConfig(
|
||||
new SourceConfig("jde", "SELECT 1", null, null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
||||
Daily = new ScheduleConfig { Enabled = true },
|
||||
Hourly = new ScheduleConfig { Enabled = false }
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null);
|
||||
|
||||
config.Schedules.ShouldNotBeNull();
|
||||
config.Schedules!.Mass!.PrePurge.ShouldBeTrue();
|
||||
config.Schedules!.Hourly!.Enabled.ShouldBeFalse();
|
||||
}
|
||||
|
||||
private static PipelineConfig CreateMinimalPipelineConfig()
|
||||
{
|
||||
return new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT 1", null, null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null);
|
||||
}
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Configuration;
|
||||
|
||||
public class ScheduleConfigTests
|
||||
{
|
||||
[Fact]
|
||||
public void ScheduleConfig_DefaultValues_AreCorrect()
|
||||
{
|
||||
var config = new ScheduleConfig();
|
||||
|
||||
config.Enabled.ShouldBeTrue();
|
||||
config.IntervalMinutes.ShouldBe(0);
|
||||
config.PrePurge.ShouldBeFalse();
|
||||
config.ReIndex.ShouldBeFalse();
|
||||
config.UpdateWhen.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleConfig_WithValues_StoresCorrectly()
|
||||
{
|
||||
var config = new ScheduleConfig
|
||||
{
|
||||
Enabled = false,
|
||||
IntervalMinutes = 60,
|
||||
PrePurge = true,
|
||||
ReIndex = true,
|
||||
UpdateWhen = "src.LastUpdateDt > tgt.LastUpdateDt"
|
||||
};
|
||||
|
||||
config.Enabled.ShouldBeFalse();
|
||||
config.IntervalMinutes.ShouldBe(60);
|
||||
config.PrePurge.ShouldBeTrue();
|
||||
config.ReIndex.ShouldBeTrue();
|
||||
config.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleDefaults_HasCorrectDefaultValues()
|
||||
{
|
||||
var defaults = new ScheduleDefaults();
|
||||
|
||||
defaults.Mass.ShouldNotBeNull();
|
||||
defaults.Daily.ShouldNotBeNull();
|
||||
defaults.Hourly.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleDefaults_Mass_HasCorrectValues()
|
||||
{
|
||||
var defaults = new ScheduleDefaults();
|
||||
|
||||
defaults.Mass.Enabled.ShouldBeTrue();
|
||||
defaults.Mass.IntervalMinutes.ShouldBe(10080); // Weekly
|
||||
defaults.Mass.PrePurge.ShouldBeTrue();
|
||||
defaults.Mass.ReIndex.ShouldBeTrue();
|
||||
defaults.Mass.UpdateWhen.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleDefaults_Daily_HasCorrectValues()
|
||||
{
|
||||
var defaults = new ScheduleDefaults();
|
||||
|
||||
defaults.Daily.Enabled.ShouldBeTrue();
|
||||
defaults.Daily.IntervalMinutes.ShouldBe(1440); // Daily
|
||||
defaults.Daily.PrePurge.ShouldBeFalse();
|
||||
defaults.Daily.ReIndex.ShouldBeFalse();
|
||||
defaults.Daily.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleDefaults_Hourly_HasCorrectValues()
|
||||
{
|
||||
var defaults = new ScheduleDefaults();
|
||||
|
||||
defaults.Hourly.Enabled.ShouldBeTrue();
|
||||
defaults.Hourly.IntervalMinutes.ShouldBe(60); // Hourly
|
||||
defaults.Hourly.PrePurge.ShouldBeFalse();
|
||||
defaults.Hourly.ReIndex.ShouldBeFalse();
|
||||
defaults.Hourly.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PipelineSchedules_AllPropertiesNullable()
|
||||
{
|
||||
var schedules = new PipelineSchedules();
|
||||
|
||||
schedules.Mass.ShouldBeNull();
|
||||
schedules.Daily.ShouldBeNull();
|
||||
schedules.Hourly.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PipelineSchedules_WithValues_StoresCorrectly()
|
||||
{
|
||||
var schedules = new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig { Enabled = true },
|
||||
Hourly = new ScheduleConfig { Enabled = false }
|
||||
};
|
||||
|
||||
schedules.Mass.ShouldNotBeNull();
|
||||
schedules.Mass!.PrePurge.ShouldBeTrue();
|
||||
schedules.Daily.ShouldNotBeNull();
|
||||
schedules.Daily!.Enabled.ShouldBeTrue();
|
||||
schedules.Hourly.ShouldNotBeNull();
|
||||
schedules.Hourly!.Enabled.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeWith_WhenConfigHasNoOverrides_ReturnsDefaultValues()
|
||||
{
|
||||
var config = new ScheduleConfig();
|
||||
var defaults = new ScheduleConfig
|
||||
{
|
||||
Enabled = true,
|
||||
IntervalMinutes = 60,
|
||||
PrePurge = false,
|
||||
ReIndex = false,
|
||||
UpdateWhen = "src.LastUpdateDt > tgt.LastUpdateDt"
|
||||
};
|
||||
|
||||
var merged = config.MergeWith(defaults);
|
||||
|
||||
merged.IntervalMinutes.ShouldBe(60);
|
||||
merged.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeWith_WhenConfigHasOverrides_UsesOverrideValues()
|
||||
{
|
||||
var config = new ScheduleConfig
|
||||
{
|
||||
IntervalMinutes = 120,
|
||||
PrePurge = true,
|
||||
UpdateWhen = "custom condition"
|
||||
};
|
||||
var defaults = new ScheduleConfig
|
||||
{
|
||||
IntervalMinutes = 60,
|
||||
PrePurge = false,
|
||||
UpdateWhen = "default condition"
|
||||
};
|
||||
|
||||
var merged = config.MergeWith(defaults);
|
||||
|
||||
merged.IntervalMinutes.ShouldBe(120);
|
||||
merged.PrePurge.ShouldBeTrue();
|
||||
merged.UpdateWhen.ShouldBe("custom condition");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeWith_PreservesEnabledFromConfig()
|
||||
{
|
||||
var config = new ScheduleConfig { Enabled = false };
|
||||
var defaults = new ScheduleConfig { Enabled = true };
|
||||
|
||||
var merged = config.MergeWith(defaults);
|
||||
|
||||
merged.Enabled.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeWith_WhenIntervalZero_UsesDefaultInterval()
|
||||
{
|
||||
var config = new ScheduleConfig { IntervalMinutes = 0 };
|
||||
var defaults = new ScheduleConfig { IntervalMinutes = 1440 };
|
||||
|
||||
var merged = config.MergeWith(defaults);
|
||||
|
||||
merged.IntervalMinutes.ShouldBe(1440);
|
||||
}
|
||||
}
|
||||
@@ -30,8 +30,7 @@ public class ScheduleCheckerTests
|
||||
_pipelines = [];
|
||||
_options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
|
||||
{
|
||||
LookbackMultiplier = 3,
|
||||
DataSources = []
|
||||
LookbackMultiplier = 3
|
||||
});
|
||||
|
||||
// Setup pipeline registry to return our pipeline list
|
||||
|
||||
@@ -1,795 +0,0 @@
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Services;
|
||||
|
||||
public class EtlPipelineFactoryTests
|
||||
{
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly ILogger<EtlPipeline> _logger;
|
||||
|
||||
public EtlPipelineFactoryTests()
|
||||
{
|
||||
_connectionFactory = Substitute.For<IDbConnectionFactory>();
|
||||
_logger = NullLogger<EtlPipeline>.Instance;
|
||||
}
|
||||
|
||||
#region ForTable Tests
|
||||
|
||||
[Fact]
|
||||
public void ForTable_WithValidTable_ReturnsBuilder()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var builder = factory.ForTable("TestTable");
|
||||
|
||||
// Assert
|
||||
builder.ShouldNotBeNull();
|
||||
builder.ShouldBeAssignableTo<IEtlPipelineBuilder>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForTable_WithUnknownTable_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert
|
||||
var ex = Should.Throw<InvalidOperationException>(() => factory.ForTable("NonExistentTable"));
|
||||
ex.Message.ShouldContain("No pipeline configured for table: NonExistentTable");
|
||||
ex.Message.ShouldContain("TestTable"); // Should list available tables
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForTable_WithNullTableName_ThrowsArgumentException()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert
|
||||
Should.Throw<ArgumentException>(() => factory.ForTable(null!));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForTable_WithEmptyTableName_ThrowsArgumentException()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert
|
||||
Should.Throw<ArgumentException>(() => factory.ForTable(""));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Builder WithUpdateType Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesMass_BuildsPipeline()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("TestTable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesDaily_BuildsPipeline()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Daily)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("TestTable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesHourly_BuildsPipeline()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("TestTable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesMass_UsesMassQuery()
|
||||
{
|
||||
// Arrange - config with massQuery should use it for Mass update type
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesDaily_UsesRegularQuery()
|
||||
{
|
||||
// Arrange - Daily should use regular query with date filtering
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Daily)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesMass_AppliesPrePurgeFromScheduleConfig()
|
||||
{
|
||||
// Arrange - Mass schedule should have prePurge=true from defaults
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - should not throw and should include truncate pre-script
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesMass_AppliesReIndexFromScheduleConfig()
|
||||
{
|
||||
// Arrange - Mass schedule should have reIndex=true from defaults
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - should not throw and should include reindex post-script
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesHourly_UsesUpdateWhenFromDefaults()
|
||||
{
|
||||
// Arrange - Hourly should use updateWhen from defaults
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_DefaultMode_IsHourly()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - don't call WithUpdateType()
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.Build();
|
||||
|
||||
// Assert - should work because hourly mode is defined
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Builder WithMinimumDate Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithMinimumDate_OverridesConfigOffset()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
var customDate = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
// Act - should not throw even though we're overriding
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.WithMinimumDate(customDate)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithNullMinimumDate_UsesConfigOffset()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - null minDt means use config offset
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.WithMinimumDate(null)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Config Validation Tests
|
||||
|
||||
[Fact]
|
||||
public void Validate_ConfigMissingSchedules_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
null,
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
null, // Schedules - null means invalid
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
|
||||
// Act & Assert
|
||||
var ex = Should.Throw<InvalidOperationException>(() => CreateFactory(config));
|
||||
ex.Message.ShouldContain("must define 'schedules'");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ConfigWithRuntimeParameter_ThrowsNotSupportedException()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
null,
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Id = @Id",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["id"] = new ParameterConfig("@Id", null, "runtime", null)
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig(),
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
|
||||
// Act & Assert
|
||||
var ex = Should.Throw<NotSupportedException>(() => CreateFactory(config));
|
||||
ex.Message.ShouldContain("runtime parameter source is not yet supported");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Destination Type Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_MassMode_WithPrePurge_UsesBulkImport()
|
||||
{
|
||||
// Arrange - Mass with prePurge defaults to bulkImport
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - should use bulkImport for mass mode with prePurge
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_HourlyMode_UsesBulkMerge()
|
||||
{
|
||||
// Arrange - Hourly without prePurge uses bulkMerge
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - should use bulkMerge for hourly mode
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_BulkMergeWithoutMatchColumns_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange - bulkMerge needs matchColumns
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", null, null), // No matchColumns!
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert
|
||||
var ex = Should.Throw<InvalidOperationException>(() =>
|
||||
factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly) // Uses bulkMerge
|
||||
.Build());
|
||||
ex.Message.ShouldContain("matchColumns required for bulkMerge");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Parameter Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithOffsetParameter_CreatesSource()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithJdeJulianParameter_CreatesSource()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("jde", "SELECT * FROM Test WHERE UPMJ >= :dateUpdated",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["minDt"] = new ParameterConfig(":dateUpdated", "jdeJulian", "offset", null)
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithStaticParameter_UsesConfiguredValue()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Status = @Status",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["status"] = new ParameterConfig("@Status", null, "static", "Active")
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithStaticParameterNoValue_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Status = @Status",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["status"] = new ParameterConfig("@Status", null, "static", null) // No value!
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert - must provide minDt for parameters to be processed
|
||||
var ex = Should.Throw<InvalidOperationException>(() =>
|
||||
factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.WithMinimumDate(DateTime.UtcNow.AddDays(-1))
|
||||
.Build());
|
||||
ex.Message.ShouldContain("Static parameter '@Status' requires a value");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Script Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithPrePurge_AddsTruncateScript()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithReIndex_AddsRebuildScript()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithPreScripts_AddsConfiguredScripts()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
["EXEC sp_BeforeSync"],
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithPostScripts_AddsConfiguredScripts()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
["UPDATE TestTable SET ProcessedFlag = 1 WHERE ProcessedFlag IS NULL"])
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Connection Type Tests
|
||||
|
||||
[Theory]
|
||||
[InlineData("jde")]
|
||||
[InlineData("cms")]
|
||||
[InlineData("lotfinder")]
|
||||
public void Builder_WithValidConnectionType_BuildsPipeline(string connectionType)
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig(connectionType, "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Settings Tests
|
||||
|
||||
[Fact]
|
||||
public void Factory_WithNullSettings_UsesDefaults()
|
||||
{
|
||||
// Arrange - null settings should use defaults
|
||||
var config = new PipelinesRoot(
|
||||
null, // Null settings
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private PipelinesRoot CreateValidConfigWithSchedules()
|
||||
{
|
||||
return new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
|
||||
},
|
||||
"SELECT * FROM Test"), // MassQuery
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
}
|
||||
|
||||
private EtlPipelineFactory CreateFactory(PipelinesRoot config)
|
||||
{
|
||||
return new EtlPipelineFactory(_connectionFactory, config, _logger);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Data;
|
||||
using System.Diagnostics.Metrics;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
@@ -19,7 +20,7 @@ namespace JdeScoping.DataSync.Tests.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for TableSyncOperation.
|
||||
/// Tests that the operation correctly uses the ETL pipeline with UpdateTypes.
|
||||
/// Tests that the operation correctly uses the ETL pipeline builder.
|
||||
/// </summary>
|
||||
public class TableSyncOperationTests
|
||||
{
|
||||
@@ -48,33 +49,26 @@ public class TableSyncOperationTests
|
||||
_metrics = new DataSyncMetrics(meterFactory);
|
||||
}
|
||||
|
||||
#region WithUpdateType Tests
|
||||
#region Pipeline Builder Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_WithUpdateTypesDaily_CallsWithUpdateTypeWithDaily()
|
||||
public async Task ExecuteAsync_WithDailyUpdateType_CallsBuildWithDailyUpdateType()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Daily);
|
||||
UpdateTypes? receivedUpdateType = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedUpdateType = callInfo.Arg<UpdateTypes>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -83,35 +77,28 @@ public class TableSyncOperationTests
|
||||
// Act
|
||||
await sut.ExecuteAsync(task);
|
||||
|
||||
// Assert - Verify WithUpdateType was called with Daily (not mapped to Incremental)
|
||||
// Assert
|
||||
receivedUpdateType.ShouldBe(UpdateTypes.Daily);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_WithUpdateTypesHourly_CallsWithUpdateTypeWithHourly()
|
||||
public async Task ExecuteAsync_WithHourlyUpdateType_CallsBuildWithHourlyUpdateType()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Hourly);
|
||||
UpdateTypes? receivedUpdateType = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedUpdateType = callInfo.Arg<UpdateTypes>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -125,30 +112,23 @@ public class TableSyncOperationTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_WithUpdateTypesMass_CallsWithUpdateTypeWithMass()
|
||||
public async Task ExecuteAsync_WithMassUpdateType_CallsBuildWithMassUpdateType()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Mass);
|
||||
UpdateTypes? receivedUpdateType = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedUpdateType = callInfo.Arg<UpdateTypes>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -161,35 +141,24 @@ public class TableSyncOperationTests
|
||||
receivedUpdateType.ShouldBe(UpdateTypes.Mass);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pipeline Execution Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_CallsForTableWithCorrectTableName()
|
||||
public async Task ExecuteAsync_CallsBuildWithCorrectPipelineConfig()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("WorkOrder", UpdateTypes.Daily);
|
||||
string? receivedTableName = null;
|
||||
EtlPipelineConfig? receivedConfig = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedTableName = callInfo.Arg<string>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.Build(
|
||||
Arg.Do<EtlPipelineConfig>(c => receivedConfig = c),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -199,35 +168,29 @@ public class TableSyncOperationTests
|
||||
await sut.ExecuteAsync(task);
|
||||
|
||||
// Assert
|
||||
receivedTableName.ShouldBe("WorkOrder");
|
||||
receivedConfig.ShouldNotBeNull();
|
||||
receivedConfig.Name.ShouldBe("WorkOrder");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_CallsWithMinimumDateWithTaskMinimumDt()
|
||||
public async Task ExecuteAsync_CallsBuildWithCorrectMinimumDate()
|
||||
{
|
||||
// Arrange
|
||||
var minDt = new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc);
|
||||
var task = CreateTask("TestTable", UpdateTypes.Daily, minDt);
|
||||
DateTime? receivedMinDt = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedMinDt = callInfo.Arg<DateTime?>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Do<DateTime?>(dt => receivedMinDt = dt))
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -240,25 +203,54 @@ public class TableSyncOperationTests
|
||||
receivedMinDt.ShouldBe(minDt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_TaskWithNoPipeline_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange
|
||||
var task = new DataUpdateTask
|
||||
{
|
||||
TableName = "TestTable",
|
||||
SourceSystem = "JDE",
|
||||
SourceData = "TESTTABLE",
|
||||
UpdateType = UpdateTypes.Daily,
|
||||
Pipeline = null // No pipeline!
|
||||
};
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
_metrics);
|
||||
|
||||
// Act & Assert
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(() => sut.ExecuteAsync(task));
|
||||
ex.Message.ShouldContain("No pipeline configuration");
|
||||
ex.Message.ShouldContain("TestTable");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pipeline Execution Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_SuccessfulPipeline_CompletesUpdateAsSuccess()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Daily);
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline(totalRows: 100);
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -295,15 +287,14 @@ public class TableSyncOperationTests
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -324,20 +315,17 @@ public class TableSyncOperationTests
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Daily);
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline(success: false);
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -366,15 +354,15 @@ public class TableSyncOperationTests
|
||||
SourceData = tableName.ToUpper(),
|
||||
UpdateType = updateType,
|
||||
MinimumDt = minDt,
|
||||
Config = new DataSourceConfig
|
||||
Pipeline = new EtlPipelineConfig
|
||||
{
|
||||
TableName = tableName,
|
||||
SourceSystem = "JDE",
|
||||
SourceData = tableName.ToUpper(),
|
||||
Name = tableName,
|
||||
IsEnabled = true,
|
||||
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
|
||||
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
|
||||
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
|
||||
MassSyncIntervalMinutes = 10080,
|
||||
DailySyncIntervalMinutes = 1440,
|
||||
HourlySyncIntervalMinutes = 60,
|
||||
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Models;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using JdeScoping.DataSync.Telemetry;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -540,15 +541,15 @@ public class SyncOrchestratorTests
|
||||
SourceData = tableName.ToUpper(),
|
||||
UpdateType = updateType,
|
||||
MinimumDt = null,
|
||||
Config = new DataSourceConfig
|
||||
Pipeline = new EtlPipelineConfig
|
||||
{
|
||||
TableName = tableName,
|
||||
SourceSystem = "JDE",
|
||||
SourceData = tableName.ToUpper(),
|
||||
Name = tableName,
|
||||
IsEnabled = true,
|
||||
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
|
||||
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
|
||||
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
|
||||
MassSyncIntervalMinutes = 10080,
|
||||
DailySyncIntervalMinutes = 1440,
|
||||
HourlySyncIntervalMinutes = 60,
|
||||
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2,6 +2,10 @@ using System.Diagnostics.Metrics;
|
||||
using JdeScoping.Core.Interfaces;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.Core.Models.Search;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using EtlPipelineConfig = JdeScoping.DataSync.Configuration.EtlPipelineConfig;
|
||||
using SourceElement = JdeScoping.DataSync.Configuration.SourceElement;
|
||||
using DestinationElement = JdeScoping.DataSync.Configuration.DestinationElement;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Models;
|
||||
using JdeScoping.DataSync.Options;
|
||||
@@ -655,15 +659,15 @@ public class WorkProcessorTests
|
||||
SourceData = tableName.ToUpper(),
|
||||
UpdateType = updateType,
|
||||
MinimumDt = null,
|
||||
Config = new DataSourceConfig
|
||||
Pipeline = new EtlPipelineConfig
|
||||
{
|
||||
TableName = tableName,
|
||||
SourceSystem = "JDE",
|
||||
SourceData = tableName.ToUpper(),
|
||||
Name = tableName,
|
||||
IsEnabled = true,
|
||||
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
|
||||
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
|
||||
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
|
||||
MassSyncIntervalMinutes = 10080,
|
||||
DailySyncIntervalMinutes = 1440,
|
||||
HourlySyncIntervalMinutes = 60,
|
||||
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user