refactor(configmanager): migrate to per-file pipeline system
Align ConfigManager with DataSync's per-file pipeline format (pipeline.*.json) by reusing EtlPipelineConfig types directly, eliminating duplicate models and simplifying the codebase. Removes ~3200 lines of obsolete code.
This commit is contained in:
Joseph Doherty
@@ -1,5 +1,6 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.ConfigManager.Services;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Tests.Services;
|
||||
|
||||
@@ -47,19 +48,16 @@ public class ValidationServiceTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipelines_WithDuplicateNames_ReturnsError()
|
||||
public void ValidatePipelines_WithEmptyName_ReturnsError()
|
||||
{
|
||||
// Arrange - duplicate keys not possible in dictionary, but empty names are invalid
|
||||
var config = new PipelinesConfigModel
|
||||
// Arrange
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
{
|
||||
[""] = new PipelineModel()
|
||||
}
|
||||
[""] = new EtlPipelineConfig()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipelines(config);
|
||||
var result = _sut.ValidatePipelines(pipelines);
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
@@ -69,22 +67,140 @@ public class ValidationServiceTests
|
||||
public void ValidatePipelines_WithInvalidConnection_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesConfigModel
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
["Test"] = new EtlPipelineConfig
|
||||
{
|
||||
["Test"] = new PipelineModel
|
||||
{
|
||||
Source = new PipelineSource { Connection = "invalid" }
|
||||
}
|
||||
Source = new SourceElement { Connection = "invalid" }
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipelines(config);
|
||||
var result = _sut.ValidatePipelines(pipelines);
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Connection"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithValidConfig_ReturnsNoErrors()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "Test",
|
||||
IsEnabled = true,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM Test"
|
||||
},
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Table = "TestTable",
|
||||
MatchColumns = ["Id"]
|
||||
},
|
||||
HourlySyncIntervalMinutes = 60
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
result.Errors.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithMissingQuery_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "" },
|
||||
Destination = new DestinationElement { Table = "TestTable" },
|
||||
IsManualOnly = true
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Query"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithNoScheduleAndNotManual_ReturnsWarning()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable", MatchColumns = ["Id"] },
|
||||
IsManualOnly = false // No schedules and not manual-only
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.Warnings.ShouldContain(w => w.Contains("No sync schedule"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_ManualOnly_DoesNotRequireSchedule()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable", MatchColumns = ["Id"] },
|
||||
IsManualOnly = true
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.Warnings.ShouldNotContain(w => w.Contains("No sync schedule"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithIntervalBelowMinimum_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable" },
|
||||
HourlySyncIntervalMinutes = 5 // Below minimum of 15
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Hourly sync interval"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithNoMatchColumns_ReturnsWarning()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable", MatchColumns = [] },
|
||||
IsManualOnly = true
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.Warnings.ShouldContain(w => w.Contains("No MatchColumns"));
|
||||
}
|
||||
}
|
||||
|
||||
-287
@@ -1,287 +0,0 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.ConfigManager.ViewModels.Forms;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Tests.ViewModels.Forms;
|
||||
|
||||
public class PipelineFormViewModelTests
|
||||
{
|
||||
[Fact]
|
||||
public void Constructor_InitializesFromModel()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Source = new PipelineSource
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM Test"
|
||||
},
|
||||
Destination = new PipelineDestination
|
||||
{
|
||||
Table = "TestTable",
|
||||
MatchColumns = ["Id", "Name"]
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var sut = new PipelineFormViewModel("TestPipeline", model, () => { });
|
||||
|
||||
// Assert
|
||||
sut.Name.ShouldBe("TestPipeline");
|
||||
sut.Connection.ShouldBe("jde");
|
||||
sut.Query.ShouldBe("SELECT * FROM Test");
|
||||
sut.DestinationTable.ShouldBe("TestTable");
|
||||
sut.MatchColumnsText.ShouldBe("Id\nName");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PropertyChange_UpdatesModelAndInvokesOnChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel();
|
||||
var changedInvoked = false;
|
||||
var sut = new PipelineFormViewModel("Test", model, () => changedInvoked = true);
|
||||
|
||||
// Act
|
||||
sut.Connection = "cms";
|
||||
|
||||
// Assert
|
||||
model.Source.Connection.ShouldBe("cms");
|
||||
changedInvoked.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MatchColumnsText_SplitsIntoArray()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel();
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Act
|
||||
sut.MatchColumnsText = "Col1\nCol2\nCol3";
|
||||
|
||||
// Assert
|
||||
model.Destination.MatchColumns.Length.ShouldBe(3);
|
||||
model.Destination.MatchColumns[0].ShouldBe("Col1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Schedules_AreInitialized()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Schedules = new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleModel { Enabled = true, IntervalMinutes = 10080 }
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Assert
|
||||
sut.MassSchedule.ShouldNotBeNull();
|
||||
sut.MassSchedule.Enabled.ShouldBeTrue();
|
||||
sut.MassSchedule.IntervalMinutes.ShouldBe(10080);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullSchedules_AreInitializedToDefault()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Schedules = new PipelineSchedules
|
||||
{
|
||||
Mass = null,
|
||||
Daily = null,
|
||||
Hourly = null
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Assert
|
||||
sut.MassSchedule.ShouldNotBeNull();
|
||||
sut.DailySchedule.ShouldNotBeNull();
|
||||
sut.HourlySchedule.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExcludeFromUpdateText_JoinsAndSplits()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Destination = new PipelineDestination
|
||||
{
|
||||
ExcludeFromUpdate = ["CreatedDate", "ModifiedDate"]
|
||||
}
|
||||
};
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Assert - verify getter joins correctly
|
||||
sut.ExcludeFromUpdateText.ShouldBe("CreatedDate\nModifiedDate");
|
||||
|
||||
// Act - verify setter splits correctly
|
||||
sut.ExcludeFromUpdateText = "Col1\nCol2";
|
||||
model.Destination.ExcludeFromUpdate.Length.ShouldBe(2);
|
||||
model.Destination.ExcludeFromUpdate[0].ShouldBe("Col1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PostScriptsText_HandlesNullable()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel { PostScripts = null };
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
|
||||
// Assert - null should return empty string
|
||||
sut.PostScriptsText.ShouldBe(string.Empty);
|
||||
|
||||
// Act - set some scripts
|
||||
sut.PostScriptsText = "script1.sql\nscript2.sql";
|
||||
model.PostScripts.ShouldNotBeNull();
|
||||
model.PostScripts!.Length.ShouldBe(2);
|
||||
|
||||
// Act - clear scripts by setting empty
|
||||
sut.PostScriptsText = "";
|
||||
model.PostScripts.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MassQuery_Property_ReadsAndWrites()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel
|
||||
{
|
||||
Source = new PipelineSource
|
||||
{
|
||||
MassQuery = "SELECT * FROM Test WHERE All = 1"
|
||||
}
|
||||
};
|
||||
var changedInvoked = false;
|
||||
var sut = new PipelineFormViewModel("Test", model, () => changedInvoked = true);
|
||||
|
||||
// Assert - verify getter
|
||||
sut.MassQuery.ShouldBe("SELECT * FROM Test WHERE All = 1");
|
||||
|
||||
// Act - verify setter
|
||||
sut.MassQuery = "SELECT * FROM NewTable";
|
||||
model.Source.MassQuery.ShouldBe("SELECT * FROM NewTable");
|
||||
changedInvoked.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PropertyChange_RaisesPropertyChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel();
|
||||
var sut = new PipelineFormViewModel("Test", model, () => { });
|
||||
var propertyChangedRaised = false;
|
||||
sut.PropertyChanged += (s, e) =>
|
||||
{
|
||||
if (e.PropertyName == nameof(PipelineFormViewModel.Query))
|
||||
propertyChangedRaised = true;
|
||||
};
|
||||
|
||||
// Act
|
||||
sut.Query = "SELECT * FROM NewQuery";
|
||||
|
||||
// Assert
|
||||
propertyChangedRaised.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleChange_InvokesOnChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new PipelineModel();
|
||||
var changedInvoked = false;
|
||||
var sut = new PipelineFormViewModel("Test", model, () => changedInvoked = true);
|
||||
|
||||
// Act - change schedule property (Enabled defaults to true, so set to false)
|
||||
sut.MassSchedule.Enabled = false;
|
||||
|
||||
// Assert
|
||||
changedInvoked.ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
|
||||
public class ScheduleFormViewModelTests
|
||||
{
|
||||
[Fact]
|
||||
public void Constructor_InitializesFromModel()
|
||||
{
|
||||
// Arrange
|
||||
var model = new ScheduleModel
|
||||
{
|
||||
Enabled = true,
|
||||
IntervalMinutes = 1440,
|
||||
PrePurge = true,
|
||||
ReIndex = false
|
||||
};
|
||||
|
||||
// Act
|
||||
var sut = new ScheduleFormViewModel(model, () => { });
|
||||
|
||||
// Assert
|
||||
sut.Enabled.ShouldBeTrue();
|
||||
sut.IntervalMinutes.ShouldBe(1440);
|
||||
sut.PrePurge.ShouldBeTrue();
|
||||
sut.ReIndex.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PropertyChange_UpdatesModelAndInvokesOnChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new ScheduleModel();
|
||||
var changedInvoked = false;
|
||||
var sut = new ScheduleFormViewModel(model, () => changedInvoked = true);
|
||||
|
||||
// Act
|
||||
sut.IntervalMinutes = 120;
|
||||
|
||||
// Assert
|
||||
model.IntervalMinutes.ShouldBe(120);
|
||||
changedInvoked.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PropertyChange_RaisesPropertyChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new ScheduleModel();
|
||||
var sut = new ScheduleFormViewModel(model, () => { });
|
||||
var propertyChangedRaised = false;
|
||||
sut.PropertyChanged += (s, e) =>
|
||||
{
|
||||
if (e.PropertyName == nameof(ScheduleFormViewModel.PrePurge))
|
||||
propertyChangedRaised = true;
|
||||
};
|
||||
|
||||
// Act
|
||||
sut.PrePurge = true;
|
||||
|
||||
// Assert
|
||||
propertyChangedRaised.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SameValue_DoesNotInvokeOnChanged()
|
||||
{
|
||||
// Arrange
|
||||
var model = new ScheduleModel { Enabled = true };
|
||||
var changedInvoked = false;
|
||||
var sut = new ScheduleFormViewModel(model, () => changedInvoked = true);
|
||||
|
||||
// Act - set same value
|
||||
sut.Enabled = true;
|
||||
|
||||
// Assert
|
||||
changedInvoked.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using JdeScoping.ConfigManager.Services;
|
||||
using JdeScoping.ConfigManager.Services.SecureStore;
|
||||
using JdeScoping.ConfigManager.ViewModels;
|
||||
using JdeScoping.ConfigManager.ViewModels.Forms;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NSubstitute;
|
||||
|
||||
@@ -38,7 +39,7 @@ public class MainWindowViewModelTests
|
||||
|
||||
_validationService.ValidateAppSettings(Arg.Any<ConfigModel>())
|
||||
.Returns(new ValidationResult());
|
||||
_validationService.ValidatePipelines(Arg.Any<PipelinesConfigModel>())
|
||||
_validationService.ValidatePipelines(Arg.Any<Dictionary<string, EtlPipelineConfig>>())
|
||||
.Returns(new ValidationResult());
|
||||
}
|
||||
|
||||
@@ -167,15 +168,13 @@ public class MainWindowViewModelTests
|
||||
{
|
||||
// Arrange
|
||||
var config = new ConfigModel();
|
||||
var pipelines = new PipelinesConfigModel
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
["WorkOrders"] = new EtlPipelineConfig
|
||||
{
|
||||
["WorkOrders"] = new PipelineModel
|
||||
{
|
||||
Source = new PipelineSource { Connection = "jde", Query = "SELECT * FROM WO" },
|
||||
Destination = new PipelineDestination { Table = "WorkOrder_Curr" }
|
||||
}
|
||||
Name = "WorkOrders",
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT * FROM WO" },
|
||||
Destination = new DestinationElement { Table = "WorkOrder_Curr" }
|
||||
}
|
||||
};
|
||||
var sut = CreateViewModel();
|
||||
@@ -295,13 +294,10 @@ public class MainWindowViewModelTests
|
||||
{
|
||||
// Arrange
|
||||
var config = new ConfigModel();
|
||||
var pipelines = new PipelinesConfigModel
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
{
|
||||
["Pipeline1"] = new PipelineModel(),
|
||||
["Pipeline2"] = new PipelineModel()
|
||||
}
|
||||
["Pipeline1"] = new EtlPipelineConfig { Name = "Pipeline1" },
|
||||
["Pipeline2"] = new EtlPipelineConfig { Name = "Pipeline2" }
|
||||
};
|
||||
var sut = CreateViewModel();
|
||||
|
||||
|
||||
+50
-28
@@ -1,26 +1,42 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.ConfigManager.ViewModels.PipelineSteps;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Tests.ViewModels;
|
||||
|
||||
public class RegexTransformerViewModelTests
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
private static TransformElement CreateElement(object config)
|
||||
{
|
||||
var json = JsonSerializer.Serialize(config, JsonOptions);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
return new TransformElement
|
||||
{
|
||||
TransformType = "Regex",
|
||||
Config = doc.RootElement.Clone()
|
||||
};
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Constructor_FromModel_LoadsAllProperties()
|
||||
public void Constructor_FromElement_LoadsAllProperties()
|
||||
{
|
||||
// Arrange
|
||||
var model = new TransformerModel
|
||||
var element = CreateElement(new
|
||||
{
|
||||
Type = "Regex",
|
||||
ColumnName = "BatchID",
|
||||
Pattern = "^IIS_",
|
||||
Replacement = "",
|
||||
IgnoreCase = true,
|
||||
NonMatchBehavior = NonMatchBehavior.ReturnEmpty
|
||||
};
|
||||
columnName = "BatchID",
|
||||
pattern = "^IIS_",
|
||||
replacement = "",
|
||||
ignoreCase = true,
|
||||
nonMatchBehavior = "ReturnEmpty"
|
||||
});
|
||||
|
||||
// Act
|
||||
var vm = new RegexTransformerViewModel(model, () => { });
|
||||
var vm = new RegexTransformerViewModel(element, () => { });
|
||||
|
||||
// Assert
|
||||
Assert.Equal("BatchID", vm.ColumnName);
|
||||
@@ -32,19 +48,17 @@ public class RegexTransformerViewModelTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Constructor_FromModel_MatchExtractMode_WhenReplacementNull()
|
||||
public void Constructor_FromElement_MatchExtractMode_WhenReplacementNull()
|
||||
{
|
||||
// Arrange
|
||||
var model = new TransformerModel
|
||||
var element = CreateElement(new
|
||||
{
|
||||
Type = "Regex",
|
||||
ColumnName = "Code",
|
||||
Pattern = @"(\d+)",
|
||||
Replacement = null
|
||||
};
|
||||
columnName = "Code",
|
||||
pattern = @"(\d+)"
|
||||
});
|
||||
|
||||
// Act
|
||||
var vm = new RegexTransformerViewModel(model, () => { });
|
||||
var vm = new RegexTransformerViewModel(element, () => { });
|
||||
|
||||
// Assert
|
||||
Assert.False(vm.IsFindReplaceMode);
|
||||
@@ -66,15 +80,18 @@ public class RegexTransformerViewModelTests
|
||||
};
|
||||
|
||||
// Act
|
||||
var model = vm.ToModel();
|
||||
var element = vm.ToModel();
|
||||
|
||||
// Assert
|
||||
Assert.Equal("Regex", model.Type);
|
||||
Assert.Equal("BatchID", model.ColumnName);
|
||||
Assert.Equal("^IIS_", model.Pattern);
|
||||
Assert.Equal("", model.Replacement);
|
||||
Assert.True(model.IgnoreCase);
|
||||
Assert.Equal(NonMatchBehavior.KeepOriginal, model.NonMatchBehavior);
|
||||
Assert.Equal("Regex", element.TransformType);
|
||||
Assert.True(element.Config.HasValue);
|
||||
|
||||
// Parse the config to verify
|
||||
var config = element.Config!.Value;
|
||||
Assert.Equal("BatchID", config.GetProperty("columnName").GetString());
|
||||
Assert.Equal("^IIS_", config.GetProperty("pattern").GetString());
|
||||
Assert.Equal("", config.GetProperty("replacement").GetString());
|
||||
Assert.True(config.GetProperty("ignoreCase").GetBoolean());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -89,10 +106,15 @@ public class RegexTransformerViewModelTests
|
||||
};
|
||||
|
||||
// Act
|
||||
var model = vm.ToModel();
|
||||
var element = vm.ToModel();
|
||||
|
||||
// Assert
|
||||
Assert.Null(model.Replacement); // null indicates Match & Extract mode
|
||||
Assert.True(element.Config.HasValue);
|
||||
var config = element.Config!.Value;
|
||||
|
||||
// replacement should be null in Match & Extract mode
|
||||
Assert.True(config.TryGetProperty("replacement", out var replacement));
|
||||
Assert.Equal(JsonValueKind.Null, replacement.ValueKind);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Configuration;
|
||||
|
||||
public class PipelinesRootTests
|
||||
{
|
||||
[Fact]
|
||||
public void EffectiveScheduleDefaults_WhenNull_ReturnsDefaults()
|
||||
{
|
||||
var root = new PipelinesRoot(null, null, new Dictionary<string, PipelineConfig>());
|
||||
|
||||
var defaults = root.EffectiveScheduleDefaults;
|
||||
|
||||
defaults.ShouldNotBeNull();
|
||||
defaults.Mass.IntervalMinutes.ShouldBe(10080);
|
||||
defaults.Daily.IntervalMinutes.ShouldBe(1440);
|
||||
defaults.Hourly.IntervalMinutes.ShouldBe(60);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EffectiveScheduleDefaults_WhenProvided_ReturnsProvided()
|
||||
{
|
||||
var customDefaults = new ScheduleDefaults
|
||||
{
|
||||
Mass = new ScheduleConfig { IntervalMinutes = 20000 }
|
||||
};
|
||||
var root = new PipelinesRoot(null, customDefaults, new Dictionary<string, PipelineConfig>());
|
||||
|
||||
var defaults = root.EffectiveScheduleDefaults;
|
||||
|
||||
defaults.Mass.IntervalMinutes.ShouldBe(20000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EffectiveSettings_WhenNull_ReturnsDefaults()
|
||||
{
|
||||
var root = new PipelinesRoot(null, null, new Dictionary<string, PipelineConfig>());
|
||||
|
||||
var settings = root.EffectiveSettings;
|
||||
|
||||
settings.ShouldNotBeNull();
|
||||
settings.Timezone.ShouldBe("UTC");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EffectiveSettings_WhenProvided_ReturnsProvided()
|
||||
{
|
||||
var customSettings = new PipelineSettings("America/New_York");
|
||||
var root = new PipelinesRoot(customSettings, null, new Dictionary<string, PipelineConfig>());
|
||||
|
||||
var settings = root.EffectiveSettings;
|
||||
|
||||
settings.Timezone.ShouldBe("America/New_York");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pipelines_WhenProvided_StoresCorrectly()
|
||||
{
|
||||
var pipelines = new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = CreateMinimalPipelineConfig()
|
||||
};
|
||||
var root = new PipelinesRoot(null, null, pipelines);
|
||||
|
||||
root.Pipelines.ShouldContainKey("TestTable");
|
||||
root.Pipelines["TestTable"].Destination.Table.ShouldBe("TestTable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PipelineConfig_WithSchedules_ParsesCorrectly()
|
||||
{
|
||||
var config = new PipelineConfig(
|
||||
new SourceConfig("jde", "SELECT 1", null, null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
||||
Daily = new ScheduleConfig { Enabled = true },
|
||||
Hourly = new ScheduleConfig { Enabled = false }
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null);
|
||||
|
||||
config.Schedules.ShouldNotBeNull();
|
||||
config.Schedules!.Mass!.PrePurge.ShouldBeTrue();
|
||||
config.Schedules!.Hourly!.Enabled.ShouldBeFalse();
|
||||
}
|
||||
|
||||
private static PipelineConfig CreateMinimalPipelineConfig()
|
||||
{
|
||||
return new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT 1", null, null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null);
|
||||
}
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Configuration;
|
||||
|
||||
public class ScheduleConfigTests
|
||||
{
|
||||
[Fact]
|
||||
public void ScheduleConfig_DefaultValues_AreCorrect()
|
||||
{
|
||||
var config = new ScheduleConfig();
|
||||
|
||||
config.Enabled.ShouldBeTrue();
|
||||
config.IntervalMinutes.ShouldBe(0);
|
||||
config.PrePurge.ShouldBeFalse();
|
||||
config.ReIndex.ShouldBeFalse();
|
||||
config.UpdateWhen.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleConfig_WithValues_StoresCorrectly()
|
||||
{
|
||||
var config = new ScheduleConfig
|
||||
{
|
||||
Enabled = false,
|
||||
IntervalMinutes = 60,
|
||||
PrePurge = true,
|
||||
ReIndex = true,
|
||||
UpdateWhen = "src.LastUpdateDt > tgt.LastUpdateDt"
|
||||
};
|
||||
|
||||
config.Enabled.ShouldBeFalse();
|
||||
config.IntervalMinutes.ShouldBe(60);
|
||||
config.PrePurge.ShouldBeTrue();
|
||||
config.ReIndex.ShouldBeTrue();
|
||||
config.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleDefaults_HasCorrectDefaultValues()
|
||||
{
|
||||
var defaults = new ScheduleDefaults();
|
||||
|
||||
defaults.Mass.ShouldNotBeNull();
|
||||
defaults.Daily.ShouldNotBeNull();
|
||||
defaults.Hourly.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleDefaults_Mass_HasCorrectValues()
|
||||
{
|
||||
var defaults = new ScheduleDefaults();
|
||||
|
||||
defaults.Mass.Enabled.ShouldBeTrue();
|
||||
defaults.Mass.IntervalMinutes.ShouldBe(10080); // Weekly
|
||||
defaults.Mass.PrePurge.ShouldBeTrue();
|
||||
defaults.Mass.ReIndex.ShouldBeTrue();
|
||||
defaults.Mass.UpdateWhen.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleDefaults_Daily_HasCorrectValues()
|
||||
{
|
||||
var defaults = new ScheduleDefaults();
|
||||
|
||||
defaults.Daily.Enabled.ShouldBeTrue();
|
||||
defaults.Daily.IntervalMinutes.ShouldBe(1440); // Daily
|
||||
defaults.Daily.PrePurge.ShouldBeFalse();
|
||||
defaults.Daily.ReIndex.ShouldBeFalse();
|
||||
defaults.Daily.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScheduleDefaults_Hourly_HasCorrectValues()
|
||||
{
|
||||
var defaults = new ScheduleDefaults();
|
||||
|
||||
defaults.Hourly.Enabled.ShouldBeTrue();
|
||||
defaults.Hourly.IntervalMinutes.ShouldBe(60); // Hourly
|
||||
defaults.Hourly.PrePurge.ShouldBeFalse();
|
||||
defaults.Hourly.ReIndex.ShouldBeFalse();
|
||||
defaults.Hourly.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PipelineSchedules_AllPropertiesNullable()
|
||||
{
|
||||
var schedules = new PipelineSchedules();
|
||||
|
||||
schedules.Mass.ShouldBeNull();
|
||||
schedules.Daily.ShouldBeNull();
|
||||
schedules.Hourly.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PipelineSchedules_WithValues_StoresCorrectly()
|
||||
{
|
||||
var schedules = new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig { Enabled = true },
|
||||
Hourly = new ScheduleConfig { Enabled = false }
|
||||
};
|
||||
|
||||
schedules.Mass.ShouldNotBeNull();
|
||||
schedules.Mass!.PrePurge.ShouldBeTrue();
|
||||
schedules.Daily.ShouldNotBeNull();
|
||||
schedules.Daily!.Enabled.ShouldBeTrue();
|
||||
schedules.Hourly.ShouldNotBeNull();
|
||||
schedules.Hourly!.Enabled.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeWith_WhenConfigHasNoOverrides_ReturnsDefaultValues()
|
||||
{
|
||||
var config = new ScheduleConfig();
|
||||
var defaults = new ScheduleConfig
|
||||
{
|
||||
Enabled = true,
|
||||
IntervalMinutes = 60,
|
||||
PrePurge = false,
|
||||
ReIndex = false,
|
||||
UpdateWhen = "src.LastUpdateDt > tgt.LastUpdateDt"
|
||||
};
|
||||
|
||||
var merged = config.MergeWith(defaults);
|
||||
|
||||
merged.IntervalMinutes.ShouldBe(60);
|
||||
merged.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeWith_WhenConfigHasOverrides_UsesOverrideValues()
|
||||
{
|
||||
var config = new ScheduleConfig
|
||||
{
|
||||
IntervalMinutes = 120,
|
||||
PrePurge = true,
|
||||
UpdateWhen = "custom condition"
|
||||
};
|
||||
var defaults = new ScheduleConfig
|
||||
{
|
||||
IntervalMinutes = 60,
|
||||
PrePurge = false,
|
||||
UpdateWhen = "default condition"
|
||||
};
|
||||
|
||||
var merged = config.MergeWith(defaults);
|
||||
|
||||
merged.IntervalMinutes.ShouldBe(120);
|
||||
merged.PrePurge.ShouldBeTrue();
|
||||
merged.UpdateWhen.ShouldBe("custom condition");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeWith_PreservesEnabledFromConfig()
|
||||
{
|
||||
var config = new ScheduleConfig { Enabled = false };
|
||||
var defaults = new ScheduleConfig { Enabled = true };
|
||||
|
||||
var merged = config.MergeWith(defaults);
|
||||
|
||||
merged.Enabled.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeWith_WhenIntervalZero_UsesDefaultInterval()
|
||||
{
|
||||
var config = new ScheduleConfig { IntervalMinutes = 0 };
|
||||
var defaults = new ScheduleConfig { IntervalMinutes = 1440 };
|
||||
|
||||
var merged = config.MergeWith(defaults);
|
||||
|
||||
merged.IntervalMinutes.ShouldBe(1440);
|
||||
}
|
||||
}
|
||||
@@ -30,8 +30,7 @@ public class ScheduleCheckerTests
|
||||
_pipelines = [];
|
||||
_options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
|
||||
{
|
||||
LookbackMultiplier = 3,
|
||||
DataSources = []
|
||||
LookbackMultiplier = 3
|
||||
});
|
||||
|
||||
// Setup pipeline registry to return our pipeline list
|
||||
|
||||
@@ -1,795 +0,0 @@
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Services;
|
||||
|
||||
public class EtlPipelineFactoryTests
|
||||
{
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly ILogger<EtlPipeline> _logger;
|
||||
|
||||
public EtlPipelineFactoryTests()
|
||||
{
|
||||
_connectionFactory = Substitute.For<IDbConnectionFactory>();
|
||||
_logger = NullLogger<EtlPipeline>.Instance;
|
||||
}
|
||||
|
||||
#region ForTable Tests
|
||||
|
||||
[Fact]
|
||||
public void ForTable_WithValidTable_ReturnsBuilder()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var builder = factory.ForTable("TestTable");
|
||||
|
||||
// Assert
|
||||
builder.ShouldNotBeNull();
|
||||
builder.ShouldBeAssignableTo<IEtlPipelineBuilder>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForTable_WithUnknownTable_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert
|
||||
var ex = Should.Throw<InvalidOperationException>(() => factory.ForTable("NonExistentTable"));
|
||||
ex.Message.ShouldContain("No pipeline configured for table: NonExistentTable");
|
||||
ex.Message.ShouldContain("TestTable"); // Should list available tables
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForTable_WithNullTableName_ThrowsArgumentException()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert
|
||||
Should.Throw<ArgumentException>(() => factory.ForTable(null!));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForTable_WithEmptyTableName_ThrowsArgumentException()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert
|
||||
Should.Throw<ArgumentException>(() => factory.ForTable(""));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Builder WithUpdateType Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesMass_BuildsPipeline()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("TestTable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesDaily_BuildsPipeline()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Daily)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("TestTable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesHourly_BuildsPipeline()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("TestTable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesMass_UsesMassQuery()
|
||||
{
|
||||
// Arrange - config with massQuery should use it for Mass update type
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesDaily_UsesRegularQuery()
|
||||
{
|
||||
// Arrange - Daily should use regular query with date filtering
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Daily)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesMass_AppliesPrePurgeFromScheduleConfig()
|
||||
{
|
||||
// Arrange - Mass schedule should have prePurge=true from defaults
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - should not throw and should include truncate pre-script
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesMass_AppliesReIndexFromScheduleConfig()
|
||||
{
|
||||
// Arrange - Mass schedule should have reIndex=true from defaults
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - should not throw and should include reindex post-script
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithUpdateTypesHourly_UsesUpdateWhenFromDefaults()
|
||||
{
|
||||
// Arrange - Hourly should use updateWhen from defaults
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_DefaultMode_IsHourly()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - don't call WithUpdateType()
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.Build();
|
||||
|
||||
// Assert - should work because hourly mode is defined
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Builder WithMinimumDate Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithMinimumDate_OverridesConfigOffset()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
var customDate = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
// Act - should not throw even though we're overriding
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.WithMinimumDate(customDate)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithNullMinimumDate_UsesConfigOffset()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - null minDt means use config offset
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.WithMinimumDate(null)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Config Validation Tests
|
||||
|
||||
[Fact]
|
||||
public void Validate_ConfigMissingSchedules_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
null,
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
null, // Schedules - null means invalid
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
|
||||
// Act & Assert
|
||||
var ex = Should.Throw<InvalidOperationException>(() => CreateFactory(config));
|
||||
ex.Message.ShouldContain("must define 'schedules'");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ConfigWithRuntimeParameter_ThrowsNotSupportedException()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
null,
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Id = @Id",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["id"] = new ParameterConfig("@Id", null, "runtime", null)
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig(),
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
|
||||
// Act & Assert
|
||||
var ex = Should.Throw<NotSupportedException>(() => CreateFactory(config));
|
||||
ex.Message.ShouldContain("runtime parameter source is not yet supported");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Destination Type Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_MassMode_WithPrePurge_UsesBulkImport()
|
||||
{
|
||||
// Arrange - Mass with prePurge defaults to bulkImport
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - should use bulkImport for mass mode with prePurge
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_HourlyMode_UsesBulkMerge()
|
||||
{
|
||||
// Arrange - Hourly without prePurge uses bulkMerge
|
||||
var config = CreateValidConfigWithSchedules();
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act - should use bulkMerge for hourly mode
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_BulkMergeWithoutMatchColumns_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange - bulkMerge needs matchColumns
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", null, null), // No matchColumns!
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert
|
||||
var ex = Should.Throw<InvalidOperationException>(() =>
|
||||
factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly) // Uses bulkMerge
|
||||
.Build());
|
||||
ex.Message.ShouldContain("matchColumns required for bulkMerge");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Parameter Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithOffsetParameter_CreatesSource()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithJdeJulianParameter_CreatesSource()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("jde", "SELECT * FROM Test WHERE UPMJ >= :dateUpdated",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["minDt"] = new ParameterConfig(":dateUpdated", "jdeJulian", "offset", null)
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithStaticParameter_UsesConfiguredValue()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Status = @Status",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["status"] = new ParameterConfig("@Status", null, "static", "Active")
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithStaticParameterNoValue_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Status = @Status",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["status"] = new ParameterConfig("@Status", null, "static", null) // No value!
|
||||
}),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act & Assert - must provide minDt for parameters to be processed
|
||||
var ex = Should.Throw<InvalidOperationException>(() =>
|
||||
factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.WithMinimumDate(DateTime.UtcNow.AddDays(-1))
|
||||
.Build());
|
||||
ex.Message.ShouldContain("Static parameter '@Status' requires a value");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Script Tests
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithPrePurge_AddsTruncateScript()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithReIndex_AddsRebuildScript()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithPreScripts_AddsConfiguredScripts()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
["EXEC sp_BeforeSync"],
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Builder_WithPostScripts_AddsConfiguredScripts()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
["UPDATE TestTable SET ProcessedFlag = 1 WHERE ProcessedFlag IS NULL"])
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Connection Type Tests
|
||||
|
||||
[Theory]
|
||||
[InlineData("jde")]
|
||||
[InlineData("cms")]
|
||||
[InlineData("lotfinder")]
|
||||
public void Builder_WithValidConnectionType_BuildsPipeline(string connectionType)
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig(connectionType, "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Mass)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Settings Tests
|
||||
|
||||
[Fact]
|
||||
public void Factory_WithNullSettings_UsesDefaults()
|
||||
{
|
||||
// Arrange - null settings should use defaults
|
||||
var config = new PipelinesRoot(
|
||||
null, // Null settings
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
var factory = CreateFactory(config);
|
||||
|
||||
// Act
|
||||
var pipeline = factory.ForTable("TestTable")
|
||||
.WithUpdateType(UpdateTypes.Hourly)
|
||||
.Build();
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private PipelinesRoot CreateValidConfigWithSchedules()
|
||||
{
|
||||
return new PipelinesRoot(
|
||||
new PipelineSettings("UTC"),
|
||||
new ScheduleDefaults(),
|
||||
new Dictionary<string, PipelineConfig>
|
||||
{
|
||||
["TestTable"] = new PipelineConfig(
|
||||
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
|
||||
new Dictionary<string, ParameterConfig>
|
||||
{
|
||||
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
|
||||
},
|
||||
"SELECT * FROM Test"), // MassQuery
|
||||
new PipelineSchedules
|
||||
{
|
||||
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
||||
Daily = new ScheduleConfig(),
|
||||
Hourly = new ScheduleConfig()
|
||||
},
|
||||
null, // Transformers
|
||||
new DestinationConfig("TestTable", ["Id"], null),
|
||||
null,
|
||||
null)
|
||||
});
|
||||
}
|
||||
|
||||
private EtlPipelineFactory CreateFactory(PipelinesRoot config)
|
||||
{
|
||||
return new EtlPipelineFactory(_connectionFactory, config, _logger);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Data;
|
||||
using System.Diagnostics.Metrics;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
@@ -19,7 +20,7 @@ namespace JdeScoping.DataSync.Tests.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for TableSyncOperation.
|
||||
/// Tests that the operation correctly uses the ETL pipeline with UpdateTypes.
|
||||
/// Tests that the operation correctly uses the ETL pipeline builder.
|
||||
/// </summary>
|
||||
public class TableSyncOperationTests
|
||||
{
|
||||
@@ -48,33 +49,26 @@ public class TableSyncOperationTests
|
||||
_metrics = new DataSyncMetrics(meterFactory);
|
||||
}
|
||||
|
||||
#region WithUpdateType Tests
|
||||
#region Pipeline Builder Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_WithUpdateTypesDaily_CallsWithUpdateTypeWithDaily()
|
||||
public async Task ExecuteAsync_WithDailyUpdateType_CallsBuildWithDailyUpdateType()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Daily);
|
||||
UpdateTypes? receivedUpdateType = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedUpdateType = callInfo.Arg<UpdateTypes>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -83,35 +77,28 @@ public class TableSyncOperationTests
|
||||
// Act
|
||||
await sut.ExecuteAsync(task);
|
||||
|
||||
// Assert - Verify WithUpdateType was called with Daily (not mapped to Incremental)
|
||||
// Assert
|
||||
receivedUpdateType.ShouldBe(UpdateTypes.Daily);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_WithUpdateTypesHourly_CallsWithUpdateTypeWithHourly()
|
||||
public async Task ExecuteAsync_WithHourlyUpdateType_CallsBuildWithHourlyUpdateType()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Hourly);
|
||||
UpdateTypes? receivedUpdateType = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedUpdateType = callInfo.Arg<UpdateTypes>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -125,30 +112,23 @@ public class TableSyncOperationTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_WithUpdateTypesMass_CallsWithUpdateTypeWithMass()
|
||||
public async Task ExecuteAsync_WithMassUpdateType_CallsBuildWithMassUpdateType()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Mass);
|
||||
UpdateTypes? receivedUpdateType = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedUpdateType = callInfo.Arg<UpdateTypes>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -161,35 +141,24 @@ public class TableSyncOperationTests
|
||||
receivedUpdateType.ShouldBe(UpdateTypes.Mass);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pipeline Execution Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_CallsForTableWithCorrectTableName()
|
||||
public async Task ExecuteAsync_CallsBuildWithCorrectPipelineConfig()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("WorkOrder", UpdateTypes.Daily);
|
||||
string? receivedTableName = null;
|
||||
EtlPipelineConfig? receivedConfig = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedTableName = callInfo.Arg<string>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.Build(
|
||||
Arg.Do<EtlPipelineConfig>(c => receivedConfig = c),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -199,35 +168,29 @@ public class TableSyncOperationTests
|
||||
await sut.ExecuteAsync(task);
|
||||
|
||||
// Assert
|
||||
receivedTableName.ShouldBe("WorkOrder");
|
||||
receivedConfig.ShouldNotBeNull();
|
||||
receivedConfig.Name.ShouldBe("WorkOrder");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_CallsWithMinimumDateWithTaskMinimumDt()
|
||||
public async Task ExecuteAsync_CallsBuildWithCorrectMinimumDate()
|
||||
{
|
||||
// Arrange
|
||||
var minDt = new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc);
|
||||
var task = CreateTask("TestTable", UpdateTypes.Daily, minDt);
|
||||
DateTime? receivedMinDt = null;
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
receivedMinDt = callInfo.Arg<DateTime?>();
|
||||
return mockBuilder;
|
||||
});
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Do<DateTime?>(dt => receivedMinDt = dt))
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -240,25 +203,54 @@ public class TableSyncOperationTests
|
||||
receivedMinDt.ShouldBe(minDt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_TaskWithNoPipeline_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Arrange
|
||||
var task = new DataUpdateTask
|
||||
{
|
||||
TableName = "TestTable",
|
||||
SourceSystem = "JDE",
|
||||
SourceData = "TESTTABLE",
|
||||
UpdateType = UpdateTypes.Daily,
|
||||
Pipeline = null // No pipeline!
|
||||
};
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
_metrics);
|
||||
|
||||
// Act & Assert
|
||||
var ex = await Should.ThrowAsync<InvalidOperationException>(() => sut.ExecuteAsync(task));
|
||||
ex.Message.ShouldContain("No pipeline configuration");
|
||||
ex.Message.ShouldContain("TestTable");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pipeline Execution Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_SuccessfulPipeline_CompletesUpdateAsSuccess()
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Daily);
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline(totalRows: 100);
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -295,15 +287,14 @@ public class TableSyncOperationTests
|
||||
var testPipeline = CreateTestPipeline();
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -324,20 +315,17 @@ public class TableSyncOperationTests
|
||||
{
|
||||
// Arrange
|
||||
var task = CreateTask("TestTable", UpdateTypes.Daily);
|
||||
|
||||
// Pre-create the test pipeline to avoid NSubstitute issues
|
||||
var testPipeline = CreateTestPipeline(success: false);
|
||||
|
||||
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
|
||||
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
|
||||
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
|
||||
mockBuilder.Build().Returns(testPipeline);
|
||||
|
||||
var mockFactory = Substitute.For<IEtlPipelineFactory>();
|
||||
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
|
||||
mockBuilder.Build(
|
||||
Arg.Any<EtlPipelineConfig>(),
|
||||
Arg.Any<UpdateTypes>(),
|
||||
Arg.Any<DateTime?>())
|
||||
.Returns(testPipeline);
|
||||
|
||||
var sut = new TableSyncOperation(
|
||||
mockFactory,
|
||||
mockBuilder,
|
||||
_updateRepository,
|
||||
_options,
|
||||
NullLogger<TableSyncOperation>.Instance,
|
||||
@@ -366,15 +354,15 @@ public class TableSyncOperationTests
|
||||
SourceData = tableName.ToUpper(),
|
||||
UpdateType = updateType,
|
||||
MinimumDt = minDt,
|
||||
Config = new DataSourceConfig
|
||||
Pipeline = new EtlPipelineConfig
|
||||
{
|
||||
TableName = tableName,
|
||||
SourceSystem = "JDE",
|
||||
SourceData = tableName.ToUpper(),
|
||||
Name = tableName,
|
||||
IsEnabled = true,
|
||||
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
|
||||
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
|
||||
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
|
||||
MassSyncIntervalMinutes = 10080,
|
||||
DailySyncIntervalMinutes = 1440,
|
||||
HourlySyncIntervalMinutes = 60,
|
||||
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Models;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using JdeScoping.DataSync.Telemetry;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -540,15 +541,15 @@ public class SyncOrchestratorTests
|
||||
SourceData = tableName.ToUpper(),
|
||||
UpdateType = updateType,
|
||||
MinimumDt = null,
|
||||
Config = new DataSourceConfig
|
||||
Pipeline = new EtlPipelineConfig
|
||||
{
|
||||
TableName = tableName,
|
||||
SourceSystem = "JDE",
|
||||
SourceData = tableName.ToUpper(),
|
||||
Name = tableName,
|
||||
IsEnabled = true,
|
||||
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
|
||||
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
|
||||
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
|
||||
MassSyncIntervalMinutes = 10080,
|
||||
DailySyncIntervalMinutes = 1440,
|
||||
HourlySyncIntervalMinutes = 60,
|
||||
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2,6 +2,10 @@ using System.Diagnostics.Metrics;
|
||||
using JdeScoping.Core.Interfaces;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.Core.Models.Search;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using EtlPipelineConfig = JdeScoping.DataSync.Configuration.EtlPipelineConfig;
|
||||
using SourceElement = JdeScoping.DataSync.Configuration.SourceElement;
|
||||
using DestinationElement = JdeScoping.DataSync.Configuration.DestinationElement;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Models;
|
||||
using JdeScoping.DataSync.Options;
|
||||
@@ -655,15 +659,15 @@ public class WorkProcessorTests
|
||||
SourceData = tableName.ToUpper(),
|
||||
UpdateType = updateType,
|
||||
MinimumDt = null,
|
||||
Config = new DataSourceConfig
|
||||
Pipeline = new EtlPipelineConfig
|
||||
{
|
||||
TableName = tableName,
|
||||
SourceSystem = "JDE",
|
||||
SourceData = tableName.ToUpper(),
|
||||
Name = tableName,
|
||||
IsEnabled = true,
|
||||
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
|
||||
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
|
||||
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
|
||||
MassSyncIntervalMinutes = 10080,
|
||||
DailySyncIntervalMinutes = 1440,
|
||||
HourlySyncIntervalMinutes = 60,
|
||||
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user