feat(etl): implement JdeDateTransformer for Julian date parsing
Add transformer that combines JDE Julian date (CYYDDD) and time (HHMMSS) columns into a single DateTime column. Includes static ParseJdeDateTime method for direct date conversion.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,136 @@
|
||||
using System.Data;
|
||||
|
||||
namespace JdeScoping.DataSync.Etl.Transformers;
|
||||
|
||||
/// <summary>
|
||||
/// A data transformer that combines JDE Julian date and time columns into a single DateTime column.
|
||||
/// JDE Julian date format is CYYDDD where C=century digit (0=1900s, 1=2000s), YY=year, DDD=day of year.
|
||||
/// Time format is HHMMSS as a decimal number.
|
||||
/// </summary>
|
||||
public class JdeDateTransformer : DataTransformerBase
|
||||
{
|
||||
private readonly string _dateColumn;
|
||||
private readonly string _timeColumn;
|
||||
private readonly string _outputColumn;
|
||||
|
||||
private int _dateOrdinal;
|
||||
private int _timeOrdinal;
|
||||
private int[]? _ordinalMap;
|
||||
private string[]? _outputNames;
|
||||
private Dictionary<string, int>? _nameToOrdinal;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string TransformerName => $"JdeDate:{_outputColumn}";
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new JdeDateTransformer that combines date and time columns.
|
||||
/// </summary>
|
||||
/// <param name="dateColumn">The name of the JDE Julian date column (CYYDDD format).</param>
|
||||
/// <param name="timeColumn">The name of the JDE time column (HHMMSS format).</param>
|
||||
/// <param name="outputColumn">The name of the output DateTime column.</param>
|
||||
public JdeDateTransformer(string dateColumn, string timeColumn, string outputColumn)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(dateColumn);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(timeColumn);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(outputColumn);
|
||||
_dateColumn = dateColumn;
|
||||
_timeColumn = timeColumn;
|
||||
_outputColumn = outputColumn;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void OnInitialize(IDataReader source)
|
||||
{
|
||||
_dateOrdinal = source.GetOrdinal(_dateColumn);
|
||||
_timeOrdinal = source.GetOrdinal(_timeColumn);
|
||||
|
||||
var ordinalList = new List<int>();
|
||||
var nameList = new List<string>();
|
||||
_nameToOrdinal = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (int i = 0; i < source.FieldCount; i++)
|
||||
{
|
||||
if (i == _timeOrdinal) continue; // Skip time column
|
||||
if (i == _dateOrdinal)
|
||||
{
|
||||
_nameToOrdinal[_outputColumn] = ordinalList.Count;
|
||||
nameList.Add(_outputColumn);
|
||||
}
|
||||
else
|
||||
{
|
||||
var name = source.GetName(i);
|
||||
_nameToOrdinal[name] = ordinalList.Count;
|
||||
nameList.Add(name);
|
||||
}
|
||||
ordinalList.Add(i);
|
||||
}
|
||||
_ordinalMap = ordinalList.ToArray();
|
||||
_outputNames = nameList.ToArray();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int GetFieldCount(IDataReader source) => _ordinalMap!.Length;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string GetName(int ordinal, IDataReader source) => _outputNames![ordinal];
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Type GetFieldType(int ordinal, IDataReader source)
|
||||
{
|
||||
var sourceOrdinal = _ordinalMap![ordinal];
|
||||
return sourceOrdinal == _dateOrdinal ? typeof(DateTime) : source.GetFieldType(sourceOrdinal);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override object GetValue(int ordinal, IDataReader source)
|
||||
{
|
||||
var sourceOrdinal = _ordinalMap![ordinal];
|
||||
if (sourceOrdinal == _dateOrdinal) return ParseJdeDateTimeFromSource(source);
|
||||
return source.GetValue(sourceOrdinal);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int GetOrdinal(string name, IDataReader source)
|
||||
{
|
||||
if (_nameToOrdinal!.TryGetValue(name, out var ordinal)) return ordinal;
|
||||
throw new IndexOutOfRangeException($"Column '{name}' not found.");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool IsDBNull(int ordinal, IDataReader source)
|
||||
{
|
||||
var sourceOrdinal = _ordinalMap![ordinal];
|
||||
if (sourceOrdinal == _dateOrdinal) return source.IsDBNull(_dateOrdinal);
|
||||
return source.IsDBNull(sourceOrdinal);
|
||||
}
|
||||
|
||||
private object ParseJdeDateTimeFromSource(IDataReader source)
|
||||
{
|
||||
if (source.IsDBNull(_dateOrdinal)) return DBNull.Value;
|
||||
var julianDate = Convert.ToDecimal(source.GetValue(_dateOrdinal));
|
||||
var timeValue = source.IsDBNull(_timeOrdinal) ? 0m : Convert.ToDecimal(source.GetValue(_timeOrdinal));
|
||||
return ParseJdeDateTime(julianDate, timeValue);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses a JDE Julian date and time into a DateTime.
|
||||
/// </summary>
|
||||
/// <param name="julianDate">The JDE Julian date in CYYDDD format.</param>
|
||||
/// <param name="time">The time in HHMMSS format.</param>
|
||||
/// <returns>The parsed DateTime.</returns>
|
||||
public static DateTime ParseJdeDateTime(decimal julianDate, decimal time)
|
||||
{
|
||||
var dateInt = (int)julianDate;
|
||||
var century = dateInt / 100000;
|
||||
var year = (dateInt / 1000) % 100;
|
||||
var dayOfYear = dateInt % 1000;
|
||||
var fullYear = (century == 0 ? 1900 : 2000) + year;
|
||||
var date = new DateTime(fullYear, 1, 1).AddDays(dayOfYear - 1);
|
||||
|
||||
var timeInt = (int)time;
|
||||
var hours = timeInt / 10000;
|
||||
var minutes = (timeInt / 100) % 100;
|
||||
var seconds = timeInt % 100;
|
||||
return date.AddHours(hours).AddMinutes(minutes).AddSeconds(seconds);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
using System.Data;
|
||||
using JdeScoping.DataSync.Etl.Transformers;
|
||||
using NSubstitute;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Etl.Transformers;
|
||||
|
||||
public class JdeDateTransformerTests
|
||||
{
|
||||
[Fact]
|
||||
public void FieldCount_ReducedByOne()
|
||||
{
|
||||
var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" });
|
||||
var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt");
|
||||
var reader = transformer.Transform(source);
|
||||
Assert.Equal(3, reader.FieldCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetName_DateColumnRenamed_TimeColumnRemoved()
|
||||
{
|
||||
var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" });
|
||||
var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt");
|
||||
var reader = transformer.Transform(source);
|
||||
Assert.Equal("Id", reader.GetName(0));
|
||||
Assert.Equal("UpdatedAt", reader.GetName(1));
|
||||
Assert.Equal("Name", reader.GetName(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetValue_ParsesJulianDateAndTime()
|
||||
{
|
||||
// Julian date 124001 = Jan 1, 2024 (century digit 1 = 2000s, year 24, day 001)
|
||||
// Time 120000 = 12:00:00
|
||||
var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" });
|
||||
source.Read().Returns(true);
|
||||
var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt");
|
||||
var reader = transformer.Transform(source);
|
||||
reader.Read();
|
||||
var expectedDate = new DateTime(2024, 1, 1, 12, 0, 0);
|
||||
Assert.Equal(expectedDate, reader.GetValue(1));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetValue_NullDate_ReturnsDbNull()
|
||||
{
|
||||
var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, DBNull.Value, DBNull.Value, "Test" });
|
||||
source.Read().Returns(true);
|
||||
source.IsDBNull(1).Returns(true);
|
||||
source.IsDBNull(2).Returns(true);
|
||||
var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt");
|
||||
var reader = transformer.Transform(source);
|
||||
reader.Read();
|
||||
Assert.Equal(DBNull.Value, reader.GetValue(1));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetFieldType_DateColumn_ReturnsDateTime()
|
||||
{
|
||||
var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" });
|
||||
var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt");
|
||||
var reader = transformer.Transform(source);
|
||||
Assert.Equal(typeof(DateTime), reader.GetFieldType(1));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetOrdinal_NewDateColumn_ReturnsCorrectOrdinal()
|
||||
{
|
||||
var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" });
|
||||
var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt");
|
||||
var reader = transformer.Transform(source);
|
||||
Assert.Equal(1, reader.GetOrdinal("UpdatedAt"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParseJdeDateTime_VariousDates()
|
||||
{
|
||||
// Test the static parsing method
|
||||
Assert.Equal(new DateTime(2024, 1, 1, 12, 0, 0), JdeDateTransformer.ParseJdeDateTime(124001m, 120000m));
|
||||
Assert.Equal(new DateTime(2023, 12, 31, 23, 59, 59), JdeDateTransformer.ParseJdeDateTime(123365m, 235959m));
|
||||
Assert.Equal(new DateTime(1999, 6, 15, 0, 0, 0), JdeDateTransformer.ParseJdeDateTime(99166m, 0m));
|
||||
}
|
||||
|
||||
private static IDataReader CreateMockReader(string[] columns, object[] values)
|
||||
{
|
||||
var reader = Substitute.For<IDataReader>();
|
||||
reader.FieldCount.Returns(columns.Length);
|
||||
for (int i = 0; i < columns.Length; i++)
|
||||
{
|
||||
var index = i;
|
||||
reader.GetName(index).Returns(columns[index]);
|
||||
reader.GetOrdinal(columns[index]).Returns(index);
|
||||
reader.GetValue(index).Returns(values[index]);
|
||||
reader.IsDBNull(index).Returns(values[index] == DBNull.Value);
|
||||
reader.GetFieldType(index).Returns(values[index]?.GetType() ?? typeof(object));
|
||||
}
|
||||
return reader;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user