Joseph Doherty
c814a7294b
- Delete SyncMode.cs enum file - Remove SyncModes property from PipelineConfig - Remove SyncModeConfig and DestinationOverride records - Remove WithMode(SyncMode) from IEtlPipelineBuilder - Remove BuildWithSyncModes() and related methods from EtlPipelineFactory - Remove syncModes sections from all pipelines in pipelines.json - Update tests to use schedules-only configuration All pipelines now require 'schedules' format (mass/daily/hourly). WithUpdateType(UpdateTypes) is the only way to set update type.
796 lines
26 KiB
C#
796 lines
26 KiB
C#
using JdeScoping.Core.Models.Enums;
|
|
using JdeScoping.DataAccess.Interfaces;
|
|
using JdeScoping.DataSync.Configuration;
|
|
using JdeScoping.DataSync.Contracts;
|
|
using JdeScoping.DataSync.Etl.Pipeline;
|
|
using JdeScoping.DataSync.Services;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using NSubstitute;
|
|
using Shouldly;
|
|
|
|
namespace JdeScoping.DataSync.Tests.Services;
|
|
|
|
public class EtlPipelineFactoryTests
|
|
{
|
|
private readonly IDbConnectionFactory _connectionFactory;
|
|
private readonly ILogger<EtlPipeline> _logger;
|
|
|
|
public EtlPipelineFactoryTests()
|
|
{
|
|
_connectionFactory = Substitute.For<IDbConnectionFactory>();
|
|
_logger = NullLogger<EtlPipeline>.Instance;
|
|
}
|
|
|
|
#region ForTable Tests
|
|
|
|
[Fact]
|
|
public void ForTable_WithValidTable_ReturnsBuilder()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var builder = factory.ForTable("TestTable");
|
|
|
|
// Assert
|
|
builder.ShouldNotBeNull();
|
|
builder.ShouldBeAssignableTo<IEtlPipelineBuilder>();
|
|
}
|
|
|
|
[Fact]
|
|
public void ForTable_WithUnknownTable_ThrowsInvalidOperationException()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act & Assert
|
|
var ex = Should.Throw<InvalidOperationException>(() => factory.ForTable("NonExistentTable"));
|
|
ex.Message.ShouldContain("No pipeline configured for table: NonExistentTable");
|
|
ex.Message.ShouldContain("TestTable"); // Should list available tables
|
|
}
|
|
|
|
[Fact]
|
|
public void ForTable_WithNullTableName_ThrowsArgumentException()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act & Assert
|
|
Should.Throw<ArgumentException>(() => factory.ForTable(null!));
|
|
}
|
|
|
|
[Fact]
|
|
public void ForTable_WithEmptyTableName_ThrowsArgumentException()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act & Assert
|
|
Should.Throw<ArgumentException>(() => factory.ForTable(""));
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Builder WithUpdateType Tests
|
|
|
|
[Fact]
|
|
public void Builder_WithUpdateTypesMass_BuildsPipeline()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Mass)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
pipeline.PipelineName.ShouldBe("TestTable");
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithUpdateTypesDaily_BuildsPipeline()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Daily)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
pipeline.PipelineName.ShouldBe("TestTable");
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithUpdateTypesHourly_BuildsPipeline()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
pipeline.PipelineName.ShouldBe("TestTable");
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithUpdateTypesMass_UsesMassQuery()
|
|
{
|
|
// Arrange - config with massQuery should use it for Mass update type
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Mass)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithUpdateTypesDaily_UsesRegularQuery()
|
|
{
|
|
// Arrange - Daily should use regular query with date filtering
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Daily)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithUpdateTypesMass_AppliesPrePurgeFromScheduleConfig()
|
|
{
|
|
// Arrange - Mass schedule should have prePurge=true from defaults
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act - should not throw and should include truncate pre-script
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Mass)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithUpdateTypesMass_AppliesReIndexFromScheduleConfig()
|
|
{
|
|
// Arrange - Mass schedule should have reIndex=true from defaults
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act - should not throw and should include reindex post-script
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Mass)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithUpdateTypesHourly_UsesUpdateWhenFromDefaults()
|
|
{
|
|
// Arrange - Hourly should use updateWhen from defaults
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_DefaultMode_IsHourly()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act - don't call WithUpdateType()
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.Build();
|
|
|
|
// Assert - should work because hourly mode is defined
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Builder WithMinimumDate Tests
|
|
|
|
[Fact]
|
|
public void Builder_WithMinimumDate_OverridesConfigOffset()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
var customDate = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
|
|
|
// Act - should not throw even though we're overriding
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.WithMinimumDate(customDate)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithNullMinimumDate_UsesConfigOffset()
|
|
{
|
|
// Arrange
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act - null minDt means use config offset
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.WithMinimumDate(null)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Config Validation Tests
|
|
|
|
[Fact]
|
|
public void Validate_ConfigMissingSchedules_ThrowsInvalidOperationException()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
null,
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
|
null, // Schedules - null means invalid
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
|
|
// Act & Assert
|
|
var ex = Should.Throw<InvalidOperationException>(() => CreateFactory(config));
|
|
ex.Message.ShouldContain("must define 'schedules'");
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_ConfigWithRuntimeParameter_ThrowsNotSupportedException()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
null,
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Id = @Id",
|
|
new Dictionary<string, ParameterConfig>
|
|
{
|
|
["id"] = new ParameterConfig("@Id", null, "runtime", null)
|
|
}),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig(),
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
|
|
// Act & Assert
|
|
var ex = Should.Throw<NotSupportedException>(() => CreateFactory(config));
|
|
ex.Message.ShouldContain("runtime parameter source is not yet supported");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Destination Type Tests
|
|
|
|
[Fact]
|
|
public void Builder_MassMode_WithPrePurge_UsesBulkImport()
|
|
{
|
|
// Arrange - Mass with prePurge defaults to bulkImport
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act - should use bulkImport for mass mode with prePurge
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Mass)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_HourlyMode_UsesBulkMerge()
|
|
{
|
|
// Arrange - Hourly without prePurge uses bulkMerge
|
|
var config = CreateValidConfigWithSchedules();
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act - should use bulkMerge for hourly mode
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_BulkMergeWithoutMatchColumns_ThrowsInvalidOperationException()
|
|
{
|
|
// Arrange - bulkMerge needs matchColumns
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", null, null), // No matchColumns!
|
|
null,
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act & Assert
|
|
var ex = Should.Throw<InvalidOperationException>(() =>
|
|
factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly) // Uses bulkMerge
|
|
.Build());
|
|
ex.Message.ShouldContain("matchColumns required for bulkMerge");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Parameter Tests
|
|
|
|
[Fact]
|
|
public void Builder_WithOffsetParameter_CreatesSource()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
|
|
new Dictionary<string, ParameterConfig>
|
|
{
|
|
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
|
|
}),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithJdeJulianParameter_CreatesSource()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("jde", "SELECT * FROM Test WHERE UPMJ >= :dateUpdated",
|
|
new Dictionary<string, ParameterConfig>
|
|
{
|
|
["minDt"] = new ParameterConfig(":dateUpdated", "jdeJulian", "offset", null)
|
|
}),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithStaticParameter_UsesConfiguredValue()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Status = @Status",
|
|
new Dictionary<string, ParameterConfig>
|
|
{
|
|
["status"] = new ParameterConfig("@Status", null, "static", "Active")
|
|
}),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithStaticParameterNoValue_ThrowsInvalidOperationException()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Status = @Status",
|
|
new Dictionary<string, ParameterConfig>
|
|
{
|
|
["status"] = new ParameterConfig("@Status", null, "static", null) // No value!
|
|
}),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act & Assert - must provide minDt for parameters to be processed
|
|
var ex = Should.Throw<InvalidOperationException>(() =>
|
|
factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.WithMinimumDate(DateTime.UtcNow.AddDays(-1))
|
|
.Build());
|
|
ex.Message.ShouldContain("Static parameter '@Status' requires a value");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Script Tests
|
|
|
|
[Fact]
|
|
public void Builder_WithPrePurge_AddsTruncateScript()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Mass)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithReIndex_AddsRebuildScript()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Mass)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithPreScripts_AddsConfiguredScripts()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
["EXEC sp_BeforeSync"],
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Mass)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_WithPostScripts_AddsConfiguredScripts()
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
["UPDATE TestTable SET ProcessedFlag = 1 WHERE ProcessedFlag IS NULL"])
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Connection Type Tests
|
|
|
|
[Theory]
|
|
[InlineData("jde")]
|
|
[InlineData("cms")]
|
|
[InlineData("lotfinder")]
|
|
public void Builder_WithValidConnectionType_BuildsPipeline(string connectionType)
|
|
{
|
|
// Arrange
|
|
var config = new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig(connectionType, "SELECT * FROM Test", null),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Mass)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Settings Tests
|
|
|
|
[Fact]
|
|
public void Factory_WithNullSettings_UsesDefaults()
|
|
{
|
|
// Arrange - null settings should use defaults
|
|
var config = new PipelinesRoot(
|
|
null, // Null settings
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
var factory = CreateFactory(config);
|
|
|
|
// Act
|
|
var pipeline = factory.ForTable("TestTable")
|
|
.WithUpdateType(UpdateTypes.Hourly)
|
|
.Build();
|
|
|
|
// Assert
|
|
pipeline.ShouldNotBeNull();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Helper Methods
|
|
|
|
private PipelinesRoot CreateValidConfigWithSchedules()
|
|
{
|
|
return new PipelinesRoot(
|
|
new PipelineSettings("UTC"),
|
|
new ScheduleDefaults(),
|
|
new Dictionary<string, PipelineConfig>
|
|
{
|
|
["TestTable"] = new PipelineConfig(
|
|
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
|
|
new Dictionary<string, ParameterConfig>
|
|
{
|
|
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
|
|
},
|
|
"SELECT * FROM Test"), // MassQuery
|
|
new PipelineSchedules
|
|
{
|
|
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
|
|
Daily = new ScheduleConfig(),
|
|
Hourly = new ScheduleConfig()
|
|
},
|
|
null, // Transformers
|
|
new DestinationConfig("TestTable", ["Id"], null),
|
|
null,
|
|
null)
|
|
});
|
|
}
|
|
|
|
private EtlPipelineFactory CreateFactory(PipelinesRoot config)
|
|
{
|
|
return new EtlPipelineFactory(_connectionFactory, config, _logger);
|
|
}
|
|
|
|
#endregion
|
|
}
|