From 6559672fc13800b0d6c0ce20f9724b3057789e1e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 17:26:36 -0400 Subject: [PATCH] Issue #30: implement value conversion --- .../Conversion/VariantConverterTests.cs | 183 ++++++ .../Conversion/VariantConverter.cs | 522 ++++++++++++++++++ 2 files changed, 705 insertions(+) create mode 100644 src/MxGateway.Worker.Tests/Conversion/VariantConverterTests.cs create mode 100644 src/MxGateway.Worker/Conversion/VariantConverter.cs diff --git a/src/MxGateway.Worker.Tests/Conversion/VariantConverterTests.cs b/src/MxGateway.Worker.Tests/Conversion/VariantConverterTests.cs new file mode 100644 index 0000000..712687f --- /dev/null +++ b/src/MxGateway.Worker.Tests/Conversion/VariantConverterTests.cs @@ -0,0 +1,183 @@ +using System; +using Google.Protobuf; +using MxGateway.Contracts.Proto; +using MxGateway.Worker.Bootstrap; +using MxGateway.Worker.Conversion; +using ProtobufTimestamp = Google.Protobuf.WellKnownTypes.Timestamp; + +namespace MxGateway.Worker.Tests.Conversion; + +public sealed class VariantConverterTests +{ + private readonly VariantConverter _converter = new(); + + [Theory] + [InlineData(true, MxDataType.Boolean, MxValue.KindOneofCase.BoolValue)] + [InlineData(42, MxDataType.Integer, MxValue.KindOneofCase.Int32Value)] + [InlineData(42L, MxDataType.Integer, MxValue.KindOneofCase.Int64Value)] + [InlineData(1.25f, MxDataType.Float, MxValue.KindOneofCase.FloatValue)] + [InlineData(2.5d, MxDataType.Double, MxValue.KindOneofCase.DoubleValue)] + [InlineData("value", MxDataType.String, MxValue.KindOneofCase.StringValue)] + public void Convert_WithSupportedScalar_ProjectsTypedValue( + object value, + MxDataType expectedDataType, + MxValue.KindOneofCase expectedKind) + { + MxValue converted = _converter.Convert(value); + + Assert.Equal(expectedDataType, converted.DataType); + Assert.Equal(expectedKind, converted.KindCase); + Assert.False(string.IsNullOrWhiteSpace(converted.VariantType)); + } + + [Fact] + public void Convert_WithDateTime_ProjectsTimestamp() + { + DateTime dateTime = new(2026, 4, 26, 17, 45, 0, DateTimeKind.Utc); + + MxValue converted = _converter.Convert(dateTime); + + Assert.Equal(MxDataType.Time, converted.DataType); + Assert.Equal(ProtobufTimestamp.FromDateTime(dateTime), converted.TimestampValue); + Assert.Equal("VT_DATE", converted.VariantType); + } + + [Fact] + public void Convert_WithFileTimeAndExpectedTime_ProjectsTimestamp() + { + DateTime dateTime = new(2026, 4, 26, 17, 45, 0, DateTimeKind.Utc); + + MxValue converted = _converter.Convert(dateTime.ToFileTimeUtc(), MxDataType.Time); + + Assert.Equal(MxDataType.Time, converted.DataType); + Assert.Equal(ProtobufTimestamp.FromDateTime(dateTime), converted.TimestampValue); + Assert.Equal("VT_I8", converted.VariantType); + } + + [Theory] + [InlineData(null, "VT_EMPTY")] + [InlineData(typeof(DBNull), "VT_NULL")] + public void Convert_WithNullLikeValue_PreservesNull( + object? value, + string expectedVariantType) + { + object? actualValue = value is System.Type ? DBNull.Value : value; + + MxValue converted = _converter.Convert(actualValue); + + Assert.True(converted.IsNull); + Assert.Equal(MxDataType.NoData, converted.DataType); + Assert.Equal(expectedVariantType, converted.VariantType); + Assert.Equal(MxValue.KindOneofCase.None, converted.KindCase); + } + + [Fact] + public void ConvertArray_WithSupportedArrays_ProjectsTypedValuesAndDimensions() + { + MxValue bools = _converter.Convert(new[] { true, false }); + MxValue ints = _converter.Convert(new[] { 1, 2, 3 }); + MxValue floats = _converter.Convert(new[] { 1.25f, 2.5f }); + MxValue doubles = _converter.Convert(new[] { 1.25d, 2.5d }); + MxValue strings = _converter.Convert(new[] { "one", "two" }); + MxValue times = _converter.Convert(new[] + { + new DateTime(2026, 4, 26, 17, 45, 0, DateTimeKind.Utc), + new DateTime(2026, 4, 26, 17, 46, 0, DateTimeKind.Utc), + }); + + Assert.Equal(new[] { true, false }, bools.ArrayValue.BoolValues.Values); + Assert.Equal(new[] { 1, 2, 3 }, ints.ArrayValue.Int32Values.Values); + Assert.Equal(new[] { 1.25f, 2.5f }, floats.ArrayValue.FloatValues.Values); + Assert.Equal(new[] { 1.25d, 2.5d }, doubles.ArrayValue.DoubleValues.Values); + Assert.Equal(new[] { "one", "two" }, strings.ArrayValue.StringValues.Values); + Assert.Equal(2, times.ArrayValue.TimestampValues.Values.Count); + Assert.Equal(new uint[] { 2 }, bools.ArrayValue.Dimensions); + Assert.Equal(MxDataType.Boolean, bools.ArrayValue.ElementDataType); + } + + [Fact] + public void ConvertArray_WithMultidimensionalArray_PreservesRankAndDimensions() + { + int[,] values = + { + { 1, 2, 3 }, + { 4, 5, 6 }, + }; + + MxValue converted = _converter.Convert(values); + + Assert.Equal(new uint[] { 2, 3 }, converted.ArrayValue.Dimensions); + Assert.Equal(new[] { 1, 2, 3, 4, 5, 6 }, converted.ArrayValue.Int32Values.Values); + } + + [Fact] + public void ConvertArray_WithExpectedTimeAndFileTimeValues_ProjectsTimestampArray() + { + DateTime first = new(2026, 4, 26, 17, 45, 0, DateTimeKind.Utc); + DateTime second = new(2026, 4, 26, 17, 46, 0, DateTimeKind.Utc); + + MxValue converted = _converter.Convert( + new[] { first.ToFileTimeUtc(), second.ToFileTimeUtc() }, + MxDataType.Time); + + Assert.Equal(MxDataType.Time, converted.ArrayValue.ElementDataType); + Assert.Equal( + new[] { ProtobufTimestamp.FromDateTime(first), ProtobufTimestamp.FromDateTime(second) }, + converted.ArrayValue.TimestampValues.Values); + } + + [Fact] + public void Convert_WithUnknownScalar_PreservesRawMetadata() + { + UnsupportedVariant value = new("opaque"); + + MxValue converted = _converter.Convert(value); + + Assert.Equal(MxDataType.Unknown, converted.DataType); + Assert.Equal(MxValue.KindOneofCase.RawValue, converted.KindCase); + Assert.Contains(typeof(UnsupportedVariant).FullName!, converted.VariantType); + Assert.Contains(typeof(UnsupportedVariant).FullName!, converted.RawDiagnostic); + Assert.Equal(ByteString.CopyFromUtf8("opaque"), converted.RawValue); + } + + [Fact] + public void ConvertArray_WithUnknownArray_PreservesRawMetadata() + { + UnsupportedVariant[] values = + [ + new("first"), + new("second"), + ]; + + MxValue converted = _converter.Convert(values); + + Assert.Equal(MxDataType.Unknown, converted.ArrayValue.ElementDataType); + Assert.Equal(MxArray.ValuesOneofCase.RawValues, converted.ArrayValue.ValuesCase); + Assert.Equal(new uint[] { 2 }, converted.ArrayValue.Dimensions); + Assert.Equal("first", converted.ArrayValue.RawValues.Values[0].ToStringUtf8()); + Assert.Contains(typeof(UnsupportedVariant).FullName!, converted.ArrayValue.RawDiagnostic); + } + + [Fact] + public void Redactor_WithCredentialBearingValueFields_RedactsBeforeLogging() + { + Assert.Equal(WorkerLogRedactor.RedactedValue, WorkerLogRedactor.RedactValue("credential_value", "secret")); + Assert.Equal(WorkerLogRedactor.RedactedValue, WorkerLogRedactor.RedactValue("password_value", "secret")); + Assert.Equal(WorkerLogRedactor.RedactedValue, WorkerLogRedactor.RedactValue("secured_write_token", "secret")); + } + + private sealed class UnsupportedVariant + { + private readonly string _value; + + public UnsupportedVariant(string value) + { + _value = value; + } + + public override string ToString() + { + return _value; + } + } +} diff --git a/src/MxGateway.Worker/Conversion/VariantConverter.cs b/src/MxGateway.Worker/Conversion/VariantConverter.cs new file mode 100644 index 0000000..9004c2f --- /dev/null +++ b/src/MxGateway.Worker/Conversion/VariantConverter.cs @@ -0,0 +1,522 @@ +using System; +using System.Globalization; +using Google.Protobuf; +using Google.Protobuf.WellKnownTypes; +using MxGateway.Contracts.Proto; + +namespace MxGateway.Worker.Conversion; + +public sealed class VariantConverter +{ + public MxValue Convert(object? value) + { + return Convert(value, MxDataType.Unspecified); + } + + public MxValue Convert( + object? value, + MxDataType expectedDataType) + { + if (value is null || value is DBNull) + { + return CreateNullValue(value, expectedDataType); + } + + if (value is Array array) + { + return new MxValue + { + DataType = MxDataType.Unspecified, + VariantType = CreateArrayVariantType(array), + ArrayValue = ConvertArray(array, expectedDataType), + }; + } + + return ConvertScalar(value, expectedDataType); + } + + public MxArray ConvertArray( + Array array, + MxDataType expectedElementDataType = MxDataType.Unspecified) + { + if (array is null) + { + throw new ArgumentNullException(nameof(array)); + } + + MxArray mxArray = new() + { + VariantType = CreateArrayVariantType(array), + }; + + for (int dimension = 0; dimension < array.Rank; dimension++) + { + mxArray.Dimensions.Add((uint)array.GetLength(dimension)); + } + + System.Type? elementType = array.GetType().GetElementType(); + MxDataType elementDataType = ResolveArrayElementDataType(elementType, expectedElementDataType); + mxArray.ElementDataType = elementDataType; + + switch (elementDataType) + { + case MxDataType.Boolean: + mxArray.BoolValues = ConvertBoolArray(array); + return mxArray; + + case MxDataType.Integer: + if (elementType == typeof(long) || elementType == typeof(ulong)) + { + mxArray.Int64Values = ConvertInt64Array(array); + } + else + { + mxArray.Int32Values = ConvertInt32Array(array); + } + + return mxArray; + + case MxDataType.Float: + mxArray.FloatValues = ConvertFloatArray(array); + return mxArray; + + case MxDataType.Double: + mxArray.DoubleValues = ConvertDoubleArray(array); + return mxArray; + + case MxDataType.String: + mxArray.StringValues = ConvertStringArray(array); + return mxArray; + + case MxDataType.Time: + mxArray.TimestampValues = ConvertTimestampArray(array); + return mxArray; + + default: + mxArray.ElementDataType = MxDataType.Unknown; + mxArray.RawElementDataType = (int)expectedElementDataType; + mxArray.RawDiagnostic = CreateRawDiagnostic(array); + mxArray.RawValues = ConvertRawArray(array); + return mxArray; + } + } + + private static MxValue ConvertScalar( + object value, + MxDataType expectedDataType) + { + System.Type valueType = value.GetType(); + string variantType = GetVariantTypeName(valueType); + + switch (System.Type.GetTypeCode(valueType)) + { + case TypeCode.Boolean: + return new MxValue + { + DataType = MxDataType.Boolean, + VariantType = variantType, + BoolValue = (bool)value, + }; + + case TypeCode.Byte: + case TypeCode.SByte: + case TypeCode.Int16: + case TypeCode.UInt16: + case TypeCode.Int32: + return new MxValue + { + DataType = MxDataType.Integer, + VariantType = variantType, + Int32Value = System.Convert.ToInt32(value, CultureInfo.InvariantCulture), + }; + + case TypeCode.UInt32: + case TypeCode.Int64: + return ConvertInt64Scalar(value, variantType, expectedDataType); + + case TypeCode.UInt64: + return ConvertUInt64Scalar((ulong)value, variantType, expectedDataType); + + case TypeCode.Single: + return new MxValue + { + DataType = MxDataType.Float, + VariantType = variantType, + FloatValue = (float)value, + }; + + case TypeCode.Double: + return new MxValue + { + DataType = MxDataType.Double, + VariantType = variantType, + DoubleValue = (double)value, + }; + + case TypeCode.Decimal: + return new MxValue + { + DataType = MxDataType.Double, + VariantType = variantType, + DoubleValue = System.Convert.ToDouble(value, CultureInfo.InvariantCulture), + RawDiagnostic = "Decimal value projected to double.", + }; + + case TypeCode.String: + case TypeCode.Char: + return new MxValue + { + DataType = MxDataType.String, + VariantType = variantType, + StringValue = System.Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty, + }; + + case TypeCode.DateTime: + return new MxValue + { + DataType = MxDataType.Time, + VariantType = variantType, + TimestampValue = ToTimestamp((DateTime)value), + }; + + default: + return CreateRawValue(value, expectedDataType); + } + } + + private static MxValue ConvertInt64Scalar( + object value, + string variantType, + MxDataType expectedDataType) + { + long longValue = System.Convert.ToInt64(value, CultureInfo.InvariantCulture); + if (expectedDataType == MxDataType.Time) + { + return new MxValue + { + DataType = MxDataType.Time, + VariantType = variantType, + TimestampValue = Timestamp.FromDateTime(DateTime.FromFileTimeUtc(longValue)), + }; + } + + return new MxValue + { + DataType = MxDataType.Integer, + VariantType = variantType, + Int64Value = longValue, + }; + } + + private static MxValue ConvertUInt64Scalar( + ulong value, + string variantType, + MxDataType expectedDataType) + { + if (expectedDataType == MxDataType.Time && value <= long.MaxValue) + { + return new MxValue + { + DataType = MxDataType.Time, + VariantType = variantType, + TimestampValue = Timestamp.FromDateTime(DateTime.FromFileTimeUtc((long)value)), + }; + } + + if (value <= long.MaxValue) + { + return new MxValue + { + DataType = MxDataType.Integer, + VariantType = variantType, + Int64Value = (long)value, + }; + } + + return CreateRawValue(value, expectedDataType, "UInt64 value exceeds Int64 range."); + } + + private static MxValue CreateNullValue( + object? value, + MxDataType expectedDataType) + { + return new MxValue + { + DataType = expectedDataType == MxDataType.Unspecified ? MxDataType.NoData : expectedDataType, + VariantType = value is DBNull ? "VT_NULL" : "VT_EMPTY", + IsNull = true, + }; + } + + private static MxValue CreateRawValue( + object value, + MxDataType expectedDataType, + string? diagnosticPrefix = null) + { + string diagnostic = CreateRawDiagnostic(value); + if (!string.IsNullOrWhiteSpace(diagnosticPrefix)) + { + diagnostic = $"{diagnosticPrefix} {diagnostic}"; + } + + return new MxValue + { + DataType = MxDataType.Unknown, + VariantType = GetVariantTypeName(value.GetType()), + RawDataType = (int)expectedDataType, + RawDiagnostic = diagnostic, + RawValue = ByteString.CopyFromUtf8(System.Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty), + }; + } + + private static BoolArray ConvertBoolArray(Array array) + { + BoolArray values = new(); + foreach (object? item in array) + { + values.Values.Add(item is not null && System.Convert.ToBoolean(item, CultureInfo.InvariantCulture)); + } + + return values; + } + + private static Int32Array ConvertInt32Array(Array array) + { + Int32Array values = new(); + foreach (object? item in array) + { + values.Values.Add(item is null ? 0 : System.Convert.ToInt32(item, CultureInfo.InvariantCulture)); + } + + return values; + } + + private static Int64Array ConvertInt64Array(Array array) + { + Int64Array values = new(); + foreach (object? item in array) + { + values.Values.Add(item is null ? 0 : System.Convert.ToInt64(item, CultureInfo.InvariantCulture)); + } + + return values; + } + + private static FloatArray ConvertFloatArray(Array array) + { + FloatArray values = new(); + foreach (object? item in array) + { + values.Values.Add(item is null ? 0 : System.Convert.ToSingle(item, CultureInfo.InvariantCulture)); + } + + return values; + } + + private static DoubleArray ConvertDoubleArray(Array array) + { + DoubleArray values = new(); + foreach (object? item in array) + { + values.Values.Add(item is null ? 0 : System.Convert.ToDouble(item, CultureInfo.InvariantCulture)); + } + + return values; + } + + private static StringArray ConvertStringArray(Array array) + { + StringArray values = new(); + foreach (object? item in array) + { + values.Values.Add(item is null ? string.Empty : System.Convert.ToString(item, CultureInfo.InvariantCulture) ?? string.Empty); + } + + return values; + } + + private static TimestampArray ConvertTimestampArray(Array array) + { + TimestampArray values = new(); + foreach (object? item in array) + { + if (item is null) + { + values.Values.Add(Timestamp.FromDateTime(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))); + } + else if (item is DateTime dateTime) + { + values.Values.Add(ToTimestamp(dateTime)); + } + else + { + long fileTime = System.Convert.ToInt64(item, CultureInfo.InvariantCulture); + values.Values.Add(Timestamp.FromDateTime(DateTime.FromFileTimeUtc(fileTime))); + } + } + + return values; + } + + private static RawArray ConvertRawArray(Array array) + { + RawArray values = new(); + foreach (object? item in array) + { + string rawValue = item is null + ? string.Empty + : System.Convert.ToString(item, CultureInfo.InvariantCulture) ?? string.Empty; + values.Values.Add(ByteString.CopyFromUtf8(rawValue)); + } + + return values; + } + + private static MxDataType ResolveArrayElementDataType( + System.Type? elementType, + MxDataType expectedElementDataType) + { + if (expectedElementDataType != MxDataType.Unspecified) + { + return expectedElementDataType; + } + + if (elementType == typeof(bool)) + { + return MxDataType.Boolean; + } + + if (elementType == typeof(byte) + || elementType == typeof(sbyte) + || elementType == typeof(short) + || elementType == typeof(ushort) + || elementType == typeof(int) + || elementType == typeof(uint) + || elementType == typeof(long) + || elementType == typeof(ulong)) + { + return MxDataType.Integer; + } + + if (elementType == typeof(float)) + { + return MxDataType.Float; + } + + if (elementType == typeof(double) || elementType == typeof(decimal)) + { + return MxDataType.Double; + } + + if (elementType == typeof(string) || elementType == typeof(char)) + { + return MxDataType.String; + } + + if (elementType == typeof(DateTime)) + { + return MxDataType.Time; + } + + return MxDataType.Unknown; + } + + private static Timestamp ToTimestamp(DateTime dateTime) + { + DateTime utcDateTime = dateTime.Kind switch + { + DateTimeKind.Utc => dateTime, + DateTimeKind.Local => dateTime.ToUniversalTime(), + _ => DateTime.SpecifyKind(dateTime, DateTimeKind.Utc), + }; + + return Timestamp.FromDateTime(utcDateTime); + } + + private static string CreateArrayVariantType(Array array) + { + System.Type? elementType = array.GetType().GetElementType(); + return $"SAFEARRAY({GetVariantTypeName(elementType)})"; + } + + private static string GetVariantTypeName(System.Type? type) + { + if (type is null) + { + return "VT_EMPTY"; + } + + System.Type nonNullableType = Nullable.GetUnderlyingType(type) ?? type; + if (nonNullableType == typeof(bool)) + { + return "VT_BOOL"; + } + + if (nonNullableType == typeof(byte)) + { + return "VT_UI1"; + } + + if (nonNullableType == typeof(sbyte)) + { + return "VT_I1"; + } + + if (nonNullableType == typeof(short)) + { + return "VT_I2"; + } + + if (nonNullableType == typeof(ushort)) + { + return "VT_UI2"; + } + + if (nonNullableType == typeof(int)) + { + return "VT_I4"; + } + + if (nonNullableType == typeof(uint)) + { + return "VT_UI4"; + } + + if (nonNullableType == typeof(long)) + { + return "VT_I8"; + } + + if (nonNullableType == typeof(ulong)) + { + return "VT_UI8"; + } + + if (nonNullableType == typeof(float)) + { + return "VT_R4"; + } + + if (nonNullableType == typeof(double) || nonNullableType == typeof(decimal)) + { + return "VT_R8"; + } + + if (nonNullableType == typeof(string) || nonNullableType == typeof(char)) + { + return "VT_BSTR"; + } + + if (nonNullableType == typeof(DateTime)) + { + return "VT_DATE"; + } + + return $"CLR:{nonNullableType.FullName}"; + } + + private static string CreateRawDiagnostic(object value) + { + return $"Unsupported variant projection for CLR type '{value.GetType().FullName}'."; + } +} -- 2.52.0