Joseph Doherty
6ebd78d487
- Add optional third parameter for parallelism (default: 8) - Use Parallel.ForEachAsync for concurrent file processing - Thread-safe console output with lock - Thread-safe size counters with Interlocked Usage: dotnet run -- <cache-dir> <scripts-dir> [parallelism]
327 lines
12 KiB
C#
327 lines
12 KiB
C#
using System.Data;
|
|
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
using ProtoBuf.Data;
|
|
using ZstdSharp;
|
|
|
|
if (args.Length < 2)
|
|
{
|
|
Console.WriteLine("Usage: CacheConverter <cache-directory> <scripts-directory> [parallelism]");
|
|
Console.WriteLine("Example: dotnet run -- ../../CACHED_DB_FILES ../../NEW/src/JdeScoping.Database/Scripts 8");
|
|
return 1;
|
|
}
|
|
|
|
var cacheDir = args[0];
|
|
var scriptsDir = args[1];
|
|
var parallelism = args.Length > 2 && int.TryParse(args[2], out var p) ? p : 8;
|
|
|
|
if (!Directory.Exists(cacheDir))
|
|
{
|
|
Console.WriteLine($"Error: Cache directory not found: {cacheDir}");
|
|
return 1;
|
|
}
|
|
|
|
if (!Directory.Exists(scriptsDir))
|
|
{
|
|
Console.WriteLine($"Error: Scripts directory not found: {scriptsDir}");
|
|
return 1;
|
|
}
|
|
|
|
// Map cache file base names to SQL script filenames and table names
|
|
var fileMapping = new Dictionary<string, (string ScriptFile, string TableName)>(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
["branch"] = ("003_CreateBranchTable.sql", "Branch"),
|
|
["functioncode"] = ("005_CreateFunctionCodeTable.sql", "FunctionCode"),
|
|
["item"] = ("008_CreateItemTable.sql", "Item"),
|
|
["jdeuser"] = ("009_CreateJdeUserTable.sql", "JdeUser"),
|
|
["lot"] = ("013_CreateLotTable.sql", "Lot"),
|
|
["lotusage_curr"] = ("024_CreateLotUsageCurrTable.sql", "LotUsage_Curr"),
|
|
["lotusage_hist"] = ("025_CreateLotUsageHistTable.sql", "LotUsage_Hist"),
|
|
["misdata"] = ("012_CreateMisDataTable.sql", "MisData"),
|
|
["orghierarchy"] = ("010_CreateOrgHierarchyTable.sql", "OrgHierarchy"),
|
|
["profitcenter"] = ("006_CreateProfitCenterTable.sql", "ProfitCenter"),
|
|
["routemaster"] = ("011_CreateRouteMasterTable.sql", "RouteMaster"),
|
|
["workcenter"] = ("007_CreateWorkCenterTable.sql", "WorkCenter"),
|
|
["workorder_curr"] = ("015_CreateWorkOrderCurrTable.sql", "WorkOrder_Curr"),
|
|
["workorder_hist"] = ("016_CreateWorkOrderHistTable.sql", "WorkOrder_Hist"),
|
|
["workordercomponent_curr"] = ("021_CreateWorkOrderComponentCurrTable.sql", "WorkOrderComponent_Curr"),
|
|
["workordercomponent_hist"] = ("022_CreateWorkOrderComponentHistTable.sql", "WorkOrderComponent_Hist"),
|
|
["workorderrouting"] = ("023_CreateWorkOrderRoutingTable.sql", "WorkOrderRouting"),
|
|
["workorderstep_curr"] = ("017_CreateWorkOrderStepCurrTable.sql", "WorkOrderStep_Curr"),
|
|
["workorderstep_hist"] = ("018_CreateWorkOrderStepHistTable.sql", "WorkOrderStep_Hist"),
|
|
["workordertime_curr"] = ("019_CreateWorkOrderTimeCurrTable.sql", "WorkOrderTime_Curr"),
|
|
["workordertime_hist"] = ("020_CreateWorkOrderTimeHistTable.sql", "WorkOrderTime_Hist"),
|
|
};
|
|
|
|
var jsonFiles = Directory.GetFiles(cacheDir, "*.json.zstd");
|
|
Console.WriteLine($"Found {jsonFiles.Length} JSON files to convert (parallelism: {parallelism})");
|
|
|
|
long totalOriginalSize = 0;
|
|
long totalNewSize = 0;
|
|
const int BatchSize = 10000;
|
|
var consoleLock = new object();
|
|
|
|
var options = new ParallelOptions { MaxDegreeOfParallelism = parallelism };
|
|
|
|
await Parallel.ForEachAsync(jsonFiles, options, async (jsonFile, cancellationToken) =>
|
|
{
|
|
var baseName = Path.GetFileName(jsonFile).Replace(".json.zstd", "");
|
|
var outputFile = Path.Combine(cacheDir, $"{baseName}.pb.zstd");
|
|
|
|
try
|
|
{
|
|
// Look up the SQL script for this file
|
|
if (!fileMapping.TryGetValue(baseName, out var mapping))
|
|
{
|
|
lock (consoleLock) Console.WriteLine($"{baseName}: SKIP (no SQL mapping)");
|
|
return;
|
|
}
|
|
|
|
var scriptPath = Path.Combine(scriptsDir, mapping.ScriptFile);
|
|
if (!File.Exists(scriptPath))
|
|
{
|
|
lock (consoleLock) Console.WriteLine($"{baseName}: SKIP (script not found: {mapping.ScriptFile})");
|
|
return;
|
|
}
|
|
|
|
// Parse schema from SQL script
|
|
var schema = ParseSqlSchema(scriptPath, mapping.TableName);
|
|
if (schema.Count == 0)
|
|
{
|
|
lock (consoleLock) Console.WriteLine($"{baseName}: SKIP (could not parse schema)");
|
|
return;
|
|
}
|
|
|
|
var originalSize = new FileInfo(jsonFile).Length;
|
|
Interlocked.Add(ref totalOriginalSize, originalSize);
|
|
|
|
// Create DataTable with schema from SQL
|
|
var dataTable = new DataTable(mapping.TableName);
|
|
foreach (var (colName, colType) in schema)
|
|
{
|
|
dataTable.Columns.Add(colName, colType);
|
|
}
|
|
|
|
// Stream JSON and write to protobuf in batches
|
|
await using var inputFs = new FileStream(jsonFile, FileMode.Open, FileAccess.Read, FileShare.Read, 256 * 1024, FileOptions.SequentialScan | FileOptions.Asynchronous);
|
|
await using var decompressStream = new DecompressionStream(inputFs);
|
|
await using var outputFs = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.None, 256 * 1024, FileOptions.Asynchronous);
|
|
await using var compressStream = new CompressionStream(outputFs, level: 10);
|
|
|
|
int rowCount = 0;
|
|
int batchCount = 0;
|
|
|
|
// True streaming: DeserializeAsyncEnumerable streams each array element without loading entire JSON
|
|
var jsonOptions = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
|
|
await foreach (var element in JsonSerializer.DeserializeAsyncEnumerable<JsonElement>(
|
|
decompressStream,
|
|
jsonOptions,
|
|
cancellationToken))
|
|
{
|
|
var row = dataTable.NewRow();
|
|
ReadJsonElement(element, row, dataTable);
|
|
dataTable.Rows.Add(row);
|
|
rowCount++;
|
|
|
|
// Write batch when we hit the batch size
|
|
if (dataTable.Rows.Count >= BatchSize)
|
|
{
|
|
using var reader = dataTable.CreateDataReader();
|
|
DataSerializer.Serialize(compressStream, reader);
|
|
dataTable.Clear();
|
|
batchCount++;
|
|
}
|
|
}
|
|
|
|
// Write remaining rows
|
|
if (dataTable.Rows.Count > 0)
|
|
{
|
|
using var reader = dataTable.CreateDataReader();
|
|
DataSerializer.Serialize(compressStream, reader);
|
|
batchCount++;
|
|
}
|
|
|
|
await compressStream.FlushAsync(cancellationToken);
|
|
|
|
var newSize = new FileInfo(outputFile).Length;
|
|
Interlocked.Add(ref totalNewSize, newSize);
|
|
|
|
var ratio = (double)newSize / originalSize * 100;
|
|
lock (consoleLock) Console.WriteLine($"{baseName}: OK ({rowCount:N0} rows, {batchCount} batches, {originalSize:N0} -> {newSize:N0} bytes, {ratio:F1}%)");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
lock (consoleLock) Console.WriteLine($"{baseName}: ERROR: {ex.Message}");
|
|
}
|
|
});
|
|
|
|
Console.WriteLine();
|
|
if (totalOriginalSize > 0)
|
|
{
|
|
Console.WriteLine($"Total: {totalOriginalSize:N0} -> {totalNewSize:N0} bytes ({(double)totalNewSize / totalOriginalSize * 100:F1}%)");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("No files were converted.");
|
|
}
|
|
return 0;
|
|
|
|
/// <summary>
|
|
/// Parse CREATE TABLE statement from SQL script to extract column names and types.
|
|
/// </summary>
|
|
static List<(string Name, Type Type)> ParseSqlSchema(string scriptPath, string tableName)
|
|
{
|
|
var result = new List<(string Name, Type Type)>();
|
|
var sql = File.ReadAllText(scriptPath);
|
|
|
|
// Find CREATE TABLE block - match the table name
|
|
var tablePattern = $@"CREATE\s+TABLE\s+\[dbo\]\.\[{Regex.Escape(tableName)}\]\s*\((.*?)\);";
|
|
var tableMatch = Regex.Match(sql, tablePattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
|
|
|
|
if (!tableMatch.Success)
|
|
{
|
|
return result;
|
|
}
|
|
|
|
var columnsBlock = tableMatch.Groups[1].Value;
|
|
|
|
// Parse each column definition
|
|
// Pattern: [ColumnName] TYPE [(size)] [NULL|NOT NULL]
|
|
var columnPattern = @"\[(\w+)\]\s+(VARCHAR|CHAR|BIGINT|DECIMAL|DATETIME2|BIT|INT)(?:\s*\(([^)]+)\))?";
|
|
var columnMatches = Regex.Matches(columnsBlock, columnPattern, RegexOptions.IgnoreCase);
|
|
|
|
foreach (Match match in columnMatches)
|
|
{
|
|
var columnName = match.Groups[1].Value;
|
|
var sqlType = match.Groups[2].Value.ToUpperInvariant();
|
|
|
|
// Skip CONSTRAINT lines
|
|
if (columnName.Equals("CONSTRAINT", StringComparison.OrdinalIgnoreCase))
|
|
continue;
|
|
|
|
var netType = MapSqlTypeToNet(sqlType);
|
|
result.Add((columnName, netType));
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Map SQL Server types to .NET types.
|
|
/// </summary>
|
|
static Type MapSqlTypeToNet(string sqlType) => sqlType.ToUpperInvariant() switch
|
|
{
|
|
"VARCHAR" => typeof(string),
|
|
"CHAR" => typeof(string),
|
|
"BIGINT" => typeof(long),
|
|
"INT" => typeof(int),
|
|
"DECIMAL" => typeof(decimal),
|
|
"DATETIME2" => typeof(DateTime),
|
|
"BIT" => typeof(bool),
|
|
_ => typeof(string)
|
|
};
|
|
|
|
/// <summary>
|
|
/// Read a JsonElement (object) into a DataRow.
|
|
/// </summary>
|
|
static void ReadJsonElement(JsonElement element, DataRow row, DataTable table)
|
|
{
|
|
foreach (var property in element.EnumerateObject())
|
|
{
|
|
// Find matching column (case-insensitive)
|
|
DataColumn? column = null;
|
|
foreach (DataColumn col in table.Columns)
|
|
{
|
|
if (col.ColumnName.Equals(property.Name, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
column = col;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (column == null)
|
|
{
|
|
// Skip unknown property
|
|
continue;
|
|
}
|
|
|
|
row[column] = ReadJsonElementValue(property.Value, column.DataType);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Read a JSON value from JsonElement and convert to the target .NET type.
|
|
/// </summary>
|
|
static object ReadJsonElementValue(JsonElement element, Type targetType)
|
|
{
|
|
if (element.ValueKind == JsonValueKind.Null || element.ValueKind == JsonValueKind.Undefined)
|
|
return DBNull.Value;
|
|
|
|
if (targetType == typeof(string))
|
|
{
|
|
return element.ValueKind switch
|
|
{
|
|
JsonValueKind.String => element.GetString() ?? (object)DBNull.Value,
|
|
JsonValueKind.Number => element.GetDecimal().ToString(),
|
|
JsonValueKind.True => "true",
|
|
JsonValueKind.False => "false",
|
|
_ => DBNull.Value
|
|
};
|
|
}
|
|
|
|
if (targetType == typeof(DateTime))
|
|
{
|
|
if (element.ValueKind == JsonValueKind.String)
|
|
{
|
|
var str = element.GetString();
|
|
if (str != null && DateTime.TryParse(str, out var dt))
|
|
return dt;
|
|
}
|
|
return DBNull.Value;
|
|
}
|
|
|
|
if (targetType == typeof(long))
|
|
{
|
|
return element.ValueKind switch
|
|
{
|
|
JsonValueKind.Number => element.GetInt64(),
|
|
JsonValueKind.String when long.TryParse(element.GetString(), out var val) => val,
|
|
_ => DBNull.Value
|
|
};
|
|
}
|
|
|
|
if (targetType == typeof(int))
|
|
{
|
|
return element.ValueKind switch
|
|
{
|
|
JsonValueKind.Number => element.GetInt32(),
|
|
JsonValueKind.String when int.TryParse(element.GetString(), out var val) => val,
|
|
_ => DBNull.Value
|
|
};
|
|
}
|
|
|
|
if (targetType == typeof(decimal))
|
|
{
|
|
return element.ValueKind switch
|
|
{
|
|
JsonValueKind.Number => element.GetDecimal(),
|
|
JsonValueKind.String when decimal.TryParse(element.GetString(), out var val) => val,
|
|
_ => DBNull.Value
|
|
};
|
|
}
|
|
|
|
if (targetType == typeof(bool))
|
|
{
|
|
return element.ValueKind switch
|
|
{
|
|
JsonValueKind.True => true,
|
|
JsonValueKind.False => false,
|
|
JsonValueKind.Number => element.GetInt32() != 0,
|
|
JsonValueKind.String => element.GetString()?.Equals("true", StringComparison.OrdinalIgnoreCase) ?? false,
|
|
_ => DBNull.Value
|
|
};
|
|
}
|
|
|
|
return DBNull.Value;
|
|
}
|