From 74c3f374469c034e38269a37fa6af69dd0f19d62 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 3 Jan 2026 09:16:11 -0500 Subject: [PATCH] feat(etl): implement JdeDateTransformer for Julian date parsing Add transformer that combines JDE Julian date (CYYDDD) and time (HHMMSS) columns into a single DateTime column. Includes static ParseJdeDateTime method for direct date conversion. --- .../Etl/Transformers/JdeDateTransformer.cs | 136 ++++++++++++++++++ .../Transformers/JdeDateTransformerTests.cs | 98 +++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 NEW/src/JdeScoping.DataSync/Etl/Transformers/JdeDateTransformer.cs create mode 100644 NEW/tests/JdeScoping.DataSync.Tests/Etl/Transformers/JdeDateTransformerTests.cs diff --git a/NEW/src/JdeScoping.DataSync/Etl/Transformers/JdeDateTransformer.cs b/NEW/src/JdeScoping.DataSync/Etl/Transformers/JdeDateTransformer.cs new file mode 100644 index 0000000..33b3d54 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync/Etl/Transformers/JdeDateTransformer.cs @@ -0,0 +1,136 @@ +using System.Data; + +namespace JdeScoping.DataSync.Etl.Transformers; + +/// +/// A data transformer that combines JDE Julian date and time columns into a single DateTime column. +/// JDE Julian date format is CYYDDD where C=century digit (0=1900s, 1=2000s), YY=year, DDD=day of year. +/// Time format is HHMMSS as a decimal number. +/// +public class JdeDateTransformer : DataTransformerBase +{ + private readonly string _dateColumn; + private readonly string _timeColumn; + private readonly string _outputColumn; + + private int _dateOrdinal; + private int _timeOrdinal; + private int[]? _ordinalMap; + private string[]? _outputNames; + private Dictionary? _nameToOrdinal; + + /// + public override string TransformerName => $"JdeDate:{_outputColumn}"; + + /// + /// Creates a new JdeDateTransformer that combines date and time columns. + /// + /// The name of the JDE Julian date column (CYYDDD format). + /// The name of the JDE time column (HHMMSS format). + /// The name of the output DateTime column. + public JdeDateTransformer(string dateColumn, string timeColumn, string outputColumn) + { + ArgumentException.ThrowIfNullOrWhiteSpace(dateColumn); + ArgumentException.ThrowIfNullOrWhiteSpace(timeColumn); + ArgumentException.ThrowIfNullOrWhiteSpace(outputColumn); + _dateColumn = dateColumn; + _timeColumn = timeColumn; + _outputColumn = outputColumn; + } + + /// + protected override void OnInitialize(IDataReader source) + { + _dateOrdinal = source.GetOrdinal(_dateColumn); + _timeOrdinal = source.GetOrdinal(_timeColumn); + + var ordinalList = new List(); + var nameList = new List(); + _nameToOrdinal = new Dictionary(StringComparer.OrdinalIgnoreCase); + + for (int i = 0; i < source.FieldCount; i++) + { + if (i == _timeOrdinal) continue; // Skip time column + if (i == _dateOrdinal) + { + _nameToOrdinal[_outputColumn] = ordinalList.Count; + nameList.Add(_outputColumn); + } + else + { + var name = source.GetName(i); + _nameToOrdinal[name] = ordinalList.Count; + nameList.Add(name); + } + ordinalList.Add(i); + } + _ordinalMap = ordinalList.ToArray(); + _outputNames = nameList.ToArray(); + } + + /// + public override int GetFieldCount(IDataReader source) => _ordinalMap!.Length; + + /// + public override string GetName(int ordinal, IDataReader source) => _outputNames![ordinal]; + + /// + public override Type GetFieldType(int ordinal, IDataReader source) + { + var sourceOrdinal = _ordinalMap![ordinal]; + return sourceOrdinal == _dateOrdinal ? typeof(DateTime) : source.GetFieldType(sourceOrdinal); + } + + /// + public override object GetValue(int ordinal, IDataReader source) + { + var sourceOrdinal = _ordinalMap![ordinal]; + if (sourceOrdinal == _dateOrdinal) return ParseJdeDateTimeFromSource(source); + return source.GetValue(sourceOrdinal); + } + + /// + public override int GetOrdinal(string name, IDataReader source) + { + if (_nameToOrdinal!.TryGetValue(name, out var ordinal)) return ordinal; + throw new IndexOutOfRangeException($"Column '{name}' not found."); + } + + /// + public override bool IsDBNull(int ordinal, IDataReader source) + { + var sourceOrdinal = _ordinalMap![ordinal]; + if (sourceOrdinal == _dateOrdinal) return source.IsDBNull(_dateOrdinal); + return source.IsDBNull(sourceOrdinal); + } + + private object ParseJdeDateTimeFromSource(IDataReader source) + { + if (source.IsDBNull(_dateOrdinal)) return DBNull.Value; + var julianDate = Convert.ToDecimal(source.GetValue(_dateOrdinal)); + var timeValue = source.IsDBNull(_timeOrdinal) ? 0m : Convert.ToDecimal(source.GetValue(_timeOrdinal)); + return ParseJdeDateTime(julianDate, timeValue); + } + + /// + /// Parses a JDE Julian date and time into a DateTime. + /// + /// The JDE Julian date in CYYDDD format. + /// The time in HHMMSS format. + /// The parsed DateTime. + public static DateTime ParseJdeDateTime(decimal julianDate, decimal time) + { + var dateInt = (int)julianDate; + var century = dateInt / 100000; + var year = (dateInt / 1000) % 100; + var dayOfYear = dateInt % 1000; + var fullYear = (century == 0 ? 1900 : 2000) + year; + var date = new DateTime(fullYear, 1, 1).AddDays(dayOfYear - 1); + + var timeInt = (int)time; + var hours = timeInt / 10000; + var minutes = (timeInt / 100) % 100; + var seconds = timeInt % 100; + return date.AddHours(hours).AddMinutes(minutes).AddSeconds(seconds); + } +} diff --git a/NEW/tests/JdeScoping.DataSync.Tests/Etl/Transformers/JdeDateTransformerTests.cs b/NEW/tests/JdeScoping.DataSync.Tests/Etl/Transformers/JdeDateTransformerTests.cs new file mode 100644 index 0000000..64057c2 --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Tests/Etl/Transformers/JdeDateTransformerTests.cs @@ -0,0 +1,98 @@ +using System.Data; +using JdeScoping.DataSync.Etl.Transformers; +using NSubstitute; + +namespace JdeScoping.DataSync.Tests.Etl.Transformers; + +public class JdeDateTransformerTests +{ + [Fact] + public void FieldCount_ReducedByOne() + { + var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" }); + var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt"); + var reader = transformer.Transform(source); + Assert.Equal(3, reader.FieldCount); + } + + [Fact] + public void GetName_DateColumnRenamed_TimeColumnRemoved() + { + var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" }); + var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt"); + var reader = transformer.Transform(source); + Assert.Equal("Id", reader.GetName(0)); + Assert.Equal("UpdatedAt", reader.GetName(1)); + Assert.Equal("Name", reader.GetName(2)); + } + + [Fact] + public void GetValue_ParsesJulianDateAndTime() + { + // Julian date 124001 = Jan 1, 2024 (century digit 1 = 2000s, year 24, day 001) + // Time 120000 = 12:00:00 + var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" }); + source.Read().Returns(true); + var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt"); + var reader = transformer.Transform(source); + reader.Read(); + var expectedDate = new DateTime(2024, 1, 1, 12, 0, 0); + Assert.Equal(expectedDate, reader.GetValue(1)); + } + + [Fact] + public void GetValue_NullDate_ReturnsDbNull() + { + var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, DBNull.Value, DBNull.Value, "Test" }); + source.Read().Returns(true); + source.IsDBNull(1).Returns(true); + source.IsDBNull(2).Returns(true); + var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt"); + var reader = transformer.Transform(source); + reader.Read(); + Assert.Equal(DBNull.Value, reader.GetValue(1)); + } + + [Fact] + public void GetFieldType_DateColumn_ReturnsDateTime() + { + var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" }); + var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt"); + var reader = transformer.Transform(source); + Assert.Equal(typeof(DateTime), reader.GetFieldType(1)); + } + + [Fact] + public void GetOrdinal_NewDateColumn_ReturnsCorrectOrdinal() + { + var source = CreateMockReader(new[] { "Id", "UPMJ", "TDAY", "Name" }, new object[] { 1, 124001m, 120000m, "Test" }); + var transformer = new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt"); + var reader = transformer.Transform(source); + Assert.Equal(1, reader.GetOrdinal("UpdatedAt")); + } + + [Fact] + public void ParseJdeDateTime_VariousDates() + { + // Test the static parsing method + Assert.Equal(new DateTime(2024, 1, 1, 12, 0, 0), JdeDateTransformer.ParseJdeDateTime(124001m, 120000m)); + Assert.Equal(new DateTime(2023, 12, 31, 23, 59, 59), JdeDateTransformer.ParseJdeDateTime(123365m, 235959m)); + Assert.Equal(new DateTime(1999, 6, 15, 0, 0, 0), JdeDateTransformer.ParseJdeDateTime(99166m, 0m)); + } + + private static IDataReader CreateMockReader(string[] columns, object[] values) + { + var reader = Substitute.For(); + reader.FieldCount.Returns(columns.Length); + for (int i = 0; i < columns.Length; i++) + { + var index = i; + reader.GetName(index).Returns(columns[index]); + reader.GetOrdinal(columns[index]).Returns(index); + reader.GetValue(index).Returns(values[index]); + reader.IsDBNull(index).Returns(values[index] == DBNull.Value); + reader.GetFieldType(index).Returns(values[index]?.GetType() ?? typeof(object)); + } + return reader; + } +}