From d2136cacf7ec3435bc1762e2197c6c95f936da0b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 6 Jan 2026 17:06:16 -0500 Subject: [PATCH] fix(DbExporter): fix compressed size calculation and clean up - Move file size read after streams are disposed to get accurate compressed size - Clean up definition files to use working example queries - Add .gitignore for output directory --- PLANS/2026-01-06-dbexporter-design.md | 117 ++++ PLANS/2026-01-06-dbexporter-implementation.md | 649 ++++++++++++++++++ Tools/DbExporter/.gitignore | 2 + Tools/DbExporter/DatabaseExporter.cs | 30 +- .../definitions/email-providers.json | 7 - .../{tags.json => example-sys-tables.json} | 4 +- .../definitions/example-tables.json | 7 + Tools/DbExporter/definitions/lmx-clients.json | 7 - .../definitions/lookup-data-types.json | 7 - .../lookup-scada-client-types.json | 7 - .../definitions/lookup-script-types.json | 7 - .../definitions/lookup-trigger-types.json | 7 - .../DbExporter/definitions/opcua-clients.json | 7 - .../DbExporter/definitions/sms-providers.json | 7 - .../definitions/teams-providers.json | 7 - .../definitions/template-scripts.json | 7 - 16 files changed, 795 insertions(+), 84 deletions(-) create mode 100644 PLANS/2026-01-06-dbexporter-design.md create mode 100644 PLANS/2026-01-06-dbexporter-implementation.md create mode 100644 Tools/DbExporter/.gitignore delete mode 100644 Tools/DbExporter/definitions/email-providers.json rename Tools/DbExporter/definitions/{tags.json => example-sys-tables.json} (67%) create mode 100644 Tools/DbExporter/definitions/example-tables.json delete mode 100644 Tools/DbExporter/definitions/lmx-clients.json delete mode 100644 Tools/DbExporter/definitions/lookup-data-types.json delete mode 100644 Tools/DbExporter/definitions/lookup-scada-client-types.json delete mode 100644 Tools/DbExporter/definitions/lookup-script-types.json delete mode 100644 Tools/DbExporter/definitions/lookup-trigger-types.json delete mode 100644 Tools/DbExporter/definitions/opcua-clients.json delete mode 100644 Tools/DbExporter/definitions/sms-providers.json delete mode 100644 Tools/DbExporter/definitions/teams-providers.json delete mode 100644 Tools/DbExporter/definitions/template-scripts.json diff --git a/PLANS/2026-01-06-dbexporter-design.md b/PLANS/2026-01-06-dbexporter-design.md new file mode 100644 index 0000000..3841709 --- /dev/null +++ b/PLANS/2026-01-06-dbexporter-design.md @@ -0,0 +1,117 @@ +# DbExporter Tool Design + +## Purpose + +A command-line tool that queries databases (SQL Server or Oracle) and exports results to compressed protobuf files using `protobuf-net-data` and zstd compression. + +## CLI Interface + +``` +Usage: DbExporter [options] + +Arguments: + definition-file Path to JSON definition file + +Options: + --verify Verify output (row count + schema) + --verify-full Verify output with SHA256 checksum + --help Show help +``` + +**Examples:** +```bash +# Export data +dotnet run -- ./definitions/scada-clients.json + +# Export and verify +dotnet run -- ./definitions/scada-clients.json --verify + +# Full verification with checksum +dotnet run -- ./definitions/scada-clients.json --verify-full +``` + +## Definition File Format (JSON) + +```json +{ + "providerType": "SqlServer", + "connectionString": "Server=...;Database=...;User Id=...;Password=...;", + "query": "SELECT * FROM MyTable", + "outputPath": "./output/mytable.pb.zstd", + "compressionLevel": 10 +} +``` + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `providerType` | Yes | - | `"SqlServer"` or `"Oracle"` | +| `connectionString` | Yes | - | ADO.NET connection string | +| `query` | Yes | - | SQL query to execute | +| `outputPath` | Yes | - | Output file path (.pb.zstd) | +| `compressionLevel` | No | `10` | Zstd level 1-19 (higher = smaller, slower) | + +## Core Workflow + +### Export Flow +1. Parse definition file (JSON) +2. Validate fields (provider type, connection string, query) +3. Create appropriate DbConnection (SqlConnection or OracleConnection) +4. Execute query → IDataReader +5. Serialize IDataReader → protobuf stream (via protobuf-net-data) +6. Compress protobuf stream → zstd (via ZstdSharp) +7. While writing, compute SHA256 incrementally +8. Write to output file + sidecar .sha256 file +9. Print summary: row count, file size, compression ratio + +### Verify Flow (--verify) +1. Open output file +2. Decompress zstd → protobuf stream +3. Deserialize protobuf → IDataReader +4. Loop through all rows, count them (streaming) +5. Extract schema (column names + types) +6. Print: ✓ row count, schema + +### Verify-Full Flow (--verify-full) +1. Open output file +2. Decompress zstd → stream protobuf data +3. While streaming: count rows, extract schema, compute SHA256 incrementally +4. Compare computed SHA256 to stored sidecar file +5. Print: ✓ row count, schema, checksum match/mismatch + +## Project Structure + +``` +Tools/DbExporter/ +├── DbExporter.csproj +├── Program.cs # CLI entry point, argument parsing +├── ExportDefinition.cs # JSON model for definition file +├── DatabaseExporter.cs # Core export logic +└── Verifier.cs # Verify and verify-full logic +``` + +## Dependencies + +| Package | Purpose | +|---------|---------| +| `protobuf-net-data` | Serialize IDataReader to protobuf | +| `ZstdSharp.Port` | Zstd compression | +| `Microsoft.Data.SqlClient` | SQL Server connectivity | +| `Oracle.ManagedDataAccess.Core` | Oracle connectivity | +| `System.Text.Json` | Parse definition files | + +**Target Framework:** `net10.0` + +## Testing with ScadaBridge + +**Connection:** +``` +Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true; +``` + +Definition files will be created in `Tools/DbExporter/definitions/` for ScadaBridge tables. + +**Test approach:** +1. Build the tool +2. Run export for each definition file +3. Run `--verify` to confirm row counts and schemas +4. Run `--verify-full` on at least one to confirm checksum works diff --git a/PLANS/2026-01-06-dbexporter-implementation.md b/PLANS/2026-01-06-dbexporter-implementation.md new file mode 100644 index 0000000..a708364 --- /dev/null +++ b/PLANS/2026-01-06-dbexporter-implementation.md @@ -0,0 +1,649 @@ +# DbExporter Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Build a CLI tool that exports database query results to compressed protobuf files. + +**Architecture:** Single console app with modular components for definition parsing, database export, and verification. + +**Tech Stack:** .NET 10, protobuf-net-data, ZstdSharp, Microsoft.Data.SqlClient, Oracle.ManagedDataAccess.Core + +--- + +### Task 1: Create Project Structure + +**Files:** +- Create: `Tools/DbExporter/DbExporter.csproj` +- Create: `Tools/DbExporter/ExportDefinition.cs` + +**Step 1: Create project directory** + +```bash +mkdir -p Tools/DbExporter +``` + +**Step 2: Create csproj file** + +```xml + + + + Exe + net10.0 + enable + enable + + + + + + + + + + +``` + +**Step 3: Create ExportDefinition model** + +```csharp +using System.Text.Json.Serialization; + +namespace DbExporter; + +public sealed class ExportDefinition +{ + [JsonPropertyName("providerType")] + public required string ProviderType { get; init; } + + [JsonPropertyName("connectionString")] + public required string ConnectionString { get; init; } + + [JsonPropertyName("query")] + public required string Query { get; init; } + + [JsonPropertyName("outputPath")] + public required string OutputPath { get; init; } + + [JsonPropertyName("compressionLevel")] + public int CompressionLevel { get; init; } = 10; +} +``` + +**Step 4: Verify build** + +```bash +cd Tools/DbExporter && dotnet build +``` + +**Step 5: Commit** + +```bash +git add Tools/DbExporter +git commit -m "feat(DbExporter): create project structure and definition model" +``` + +--- + +### Task 2: Implement DatabaseExporter + +**Files:** +- Create: `Tools/DbExporter/DatabaseExporter.cs` + +**Step 1: Create DatabaseExporter class** + +```csharp +using System.Data; +using System.Data.Common; +using System.Security.Cryptography; +using Microsoft.Data.SqlClient; +using Oracle.ManagedDataAccess.Client; +using ProtoBuf.Data; +using ZstdSharp; + +namespace DbExporter; + +public sealed class DatabaseExporter +{ + public record ExportResult(int RowCount, long UncompressedSize, long CompressedSize, string Sha256Hash); + + public async Task ExportAsync(ExportDefinition definition, CancellationToken cancellationToken = default) + { + // Ensure output directory exists + var outputDir = Path.GetDirectoryName(definition.OutputPath); + if (!string.IsNullOrEmpty(outputDir)) + Directory.CreateDirectory(outputDir); + + await using var connection = CreateConnection(definition.ProviderType, definition.ConnectionString); + await connection.OpenAsync(cancellationToken); + + await using var command = connection.CreateCommand(); + command.CommandText = definition.Query; + command.CommandTimeout = 0; // No timeout for large exports + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + + int rowCount = 0; + long uncompressedSize = 0; + + // Use memory stream to capture uncompressed protobuf for SHA256 + using var sha256 = SHA256.Create(); + await using var outputFile = new FileStream(definition.OutputPath, FileMode.Create, FileAccess.Write, FileShare.None, 256 * 1024); + await using var compressStream = new CompressionStream(outputFile, definition.CompressionLevel); + await using var hashStream = new CryptoStream(compressStream, sha256, CryptoStreamMode.Write); + + // Serialize to protobuf + DataSerializer.Serialize(hashStream, reader); + + // Count rows by re-reading (protobuf-net-data doesn't expose count during serialize) + // We'll track this differently - use a counting wrapper or post-verify + // For now, we serialize and then verify separately + + hashStream.FlushFinalBlock(); + uncompressedSize = hashStream.Length; + + var hash = Convert.ToHexString(sha256.Hash!).ToLowerInvariant(); + + // Write sidecar hash file + var hashFilePath = definition.OutputPath + ".sha256"; + await File.WriteAllTextAsync(hashFilePath, hash, cancellationToken); + + var compressedSize = new FileInfo(definition.OutputPath).Length; + + // Row count requires a separate pass or we estimate from verify + // Return 0 for now, verify will get accurate count + return new ExportResult(0, uncompressedSize, compressedSize, hash); + } + + private static DbConnection CreateConnection(string providerType, string connectionString) + { + return providerType.ToLowerInvariant() switch + { + "sqlserver" => new SqlConnection(connectionString), + "oracle" => new OracleConnection(connectionString), + _ => throw new ArgumentException($"Unknown provider type: {providerType}. Use 'SqlServer' or 'Oracle'.") + }; + } +} +``` + +**Step 2: Verify build** + +```bash +cd Tools/DbExporter && dotnet build +``` + +**Step 3: Commit** + +```bash +git add Tools/DbExporter/DatabaseExporter.cs +git commit -m "feat(DbExporter): implement database export with protobuf+zstd" +``` + +--- + +### Task 3: Implement Verifier + +**Files:** +- Create: `Tools/DbExporter/Verifier.cs` + +**Step 1: Create Verifier class** + +```csharp +using System.Data; +using System.Security.Cryptography; +using System.Text; +using ProtoBuf.Data; +using ZstdSharp; + +namespace DbExporter; + +public sealed class Verifier +{ + public record VerifyResult(int RowCount, List Schema, string? ComputedHash, string? ExpectedHash, bool? HashMatch); + public record ColumnInfo(string Name, Type Type); + + public VerifyResult Verify(string filePath, bool computeHash = false) + { + using var inputFile = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 256 * 1024); + using var decompressStream = new DecompressionStream(inputFile); + + Stream readStream = decompressStream; + SHA256? sha256 = null; + CryptoStream? hashStream = null; + + if (computeHash) + { + sha256 = SHA256.Create(); + hashStream = new CryptoStream(decompressStream, sha256, CryptoStreamMode.Read); + readStream = hashStream; + } + + using var reader = DataSerializer.Deserialize(readStream); + + // Extract schema + var schema = new List(); + for (int i = 0; i < reader.FieldCount; i++) + { + schema.Add(new ColumnInfo(reader.GetName(i), reader.GetFieldType(i))); + } + + // Count rows + int rowCount = 0; + while (reader.Read()) + { + rowCount++; + } + + string? computedHashStr = null; + string? expectedHash = null; + bool? hashMatch = null; + + if (computeHash && sha256 != null) + { + hashStream?.Dispose(); + computedHashStr = Convert.ToHexString(sha256.Hash!).ToLowerInvariant(); + + // Read expected hash from sidecar + var hashFilePath = filePath + ".sha256"; + if (File.Exists(hashFilePath)) + { + expectedHash = File.ReadAllText(hashFilePath).Trim().ToLowerInvariant(); + hashMatch = computedHashStr == expectedHash; + } + + sha256.Dispose(); + } + + return new VerifyResult(rowCount, schema, computedHashStr, expectedHash, hashMatch); + } + + public string FormatSchema(List schema) + { + var sb = new StringBuilder(); + foreach (var col in schema) + { + if (sb.Length > 0) sb.Append(", "); + sb.Append($"{col.Name} ({col.Type.Name})"); + } + return sb.ToString(); + } +} +``` + +**Step 2: Verify build** + +```bash +cd Tools/DbExporter && dotnet build +``` + +**Step 3: Commit** + +```bash +git add Tools/DbExporter/Verifier.cs +git commit -m "feat(DbExporter): implement verify and verify-full" +``` + +--- + +### Task 4: Implement CLI Entry Point + +**Files:** +- Create: `Tools/DbExporter/Program.cs` + +**Step 1: Create Program.cs with CLI parsing** + +```csharp +using System.Text.Json; +using DbExporter; + +if (args.Length < 1 || args.Contains("--help") || args.Contains("-h")) +{ + PrintUsage(); + return args.Contains("--help") || args.Contains("-h") ? 0 : 1; +} + +var definitionPath = args[0]; +var verify = args.Contains("--verify"); +var verifyFull = args.Contains("--verify-full"); + +if (!File.Exists(definitionPath)) +{ + Console.WriteLine($"Error: Definition file not found: {definitionPath}"); + return 1; +} + +try +{ + var json = await File.ReadAllTextAsync(definitionPath); + var definition = JsonSerializer.Deserialize(json); + + if (definition is null) + { + Console.WriteLine("Error: Failed to parse definition file."); + return 1; + } + + // Validate required fields + if (string.IsNullOrWhiteSpace(definition.ProviderType)) + { + Console.WriteLine("Error: providerType is required."); + return 1; + } + if (string.IsNullOrWhiteSpace(definition.ConnectionString)) + { + Console.WriteLine("Error: connectionString is required."); + return 1; + } + if (string.IsNullOrWhiteSpace(definition.Query)) + { + Console.WriteLine("Error: query is required."); + return 1; + } + if (string.IsNullOrWhiteSpace(definition.OutputPath)) + { + Console.WriteLine("Error: outputPath is required."); + return 1; + } + + var exporter = new DatabaseExporter(); + var verifier = new Verifier(); + + Console.WriteLine($"Exporting from {definition.ProviderType}..."); + Console.WriteLine($"Query: {Truncate(definition.Query, 80)}"); + + var result = await exporter.ExportAsync(definition); + + // Always do a quick verify to get row count + var quickVerify = verifier.Verify(definition.OutputPath, computeHash: false); + + var ratio = result.CompressedSize > 0 && quickVerify.RowCount > 0 + ? $" ({(double)result.CompressedSize / result.UncompressedSize * 100:F1}%)" + : ""; + + Console.WriteLine($"✓ Exported: {quickVerify.RowCount:N0} rows, {result.UncompressedSize:N0} → {result.CompressedSize:N0} bytes{ratio}"); + + if (verify || verifyFull) + { + Console.WriteLine(); + Console.WriteLine("Verifying..."); + + var verifyResult = verifier.Verify(definition.OutputPath, computeHash: verifyFull); + + Console.WriteLine($"✓ Verified: {verifyResult.RowCount:N0} rows"); + Console.WriteLine($"Schema: {verifier.FormatSchema(verifyResult.Schema)}"); + + if (verifyFull && verifyResult.HashMatch.HasValue) + { + if (verifyResult.HashMatch.Value) + { + Console.WriteLine($"✓ Checksum: SHA256 match ({verifyResult.ComputedHash})"); + } + else + { + Console.WriteLine($"✗ Checksum: SHA256 MISMATCH"); + Console.WriteLine($" Expected: {verifyResult.ExpectedHash}"); + Console.WriteLine($" Computed: {verifyResult.ComputedHash}"); + return 1; + } + } + } + + return 0; +} +catch (Exception ex) +{ + Console.WriteLine($"Error: {ex.Message}"); + return 1; +} + +static void PrintUsage() +{ + Console.WriteLine("Usage: DbExporter [options]"); + Console.WriteLine(); + Console.WriteLine("Arguments:"); + Console.WriteLine(" definition-file Path to JSON definition file"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --verify Verify output (row count + schema)"); + Console.WriteLine(" --verify-full Verify output with SHA256 checksum"); + Console.WriteLine(" --help Show this help"); + Console.WriteLine(); + Console.WriteLine("Definition file format:"); + Console.WriteLine(" {"); + Console.WriteLine(" \"providerType\": \"SqlServer\","); + Console.WriteLine(" \"connectionString\": \"Server=...;Database=...;\","); + Console.WriteLine(" \"query\": \"SELECT * FROM MyTable\","); + Console.WriteLine(" \"outputPath\": \"./output/mytable.pb.zstd\","); + Console.WriteLine(" \"compressionLevel\": 10"); + Console.WriteLine(" }"); +} + +static string Truncate(string value, int maxLength) +{ + if (string.IsNullOrEmpty(value)) return value; + var singleLine = value.Replace("\r", "").Replace("\n", " "); + return singleLine.Length <= maxLength ? singleLine : singleLine[..(maxLength - 3)] + "..."; +} +``` + +**Step 2: Verify build** + +```bash +cd Tools/DbExporter && dotnet build +``` + +**Step 3: Commit** + +```bash +git add Tools/DbExporter/Program.cs +git commit -m "feat(DbExporter): implement CLI entry point" +``` + +--- + +### Task 5: Fix Export Row Count Issue + +The current implementation computes row count during verify, but can't get it during export (protobuf-net-data streams without counting). Let's fix this by wrapping the IDataReader. + +**Files:** +- Create: `Tools/DbExporter/CountingDataReader.cs` +- Modify: `Tools/DbExporter/DatabaseExporter.cs` + +**Step 1: Create CountingDataReader wrapper** + +```csharp +using System.Data; + +namespace DbExporter; + +/// +/// Wraps an IDataReader to count rows as they're read. +/// +internal sealed class CountingDataReader : IDataReader +{ + private readonly IDataReader _inner; + private int _rowCount; + + public CountingDataReader(IDataReader inner) + { + _inner = inner; + } + + public int RowCount => _rowCount; + + public bool Read() + { + var result = _inner.Read(); + if (result) _rowCount++; + return result; + } + + // Delegate all other members to inner reader + public object this[int i] => _inner[i]; + public object this[string name] => _inner[name]; + public int Depth => _inner.Depth; + public bool IsClosed => _inner.IsClosed; + public int RecordsAffected => _inner.RecordsAffected; + public int FieldCount => _inner.FieldCount; + public void Close() => _inner.Close(); + public void Dispose() => _inner.Dispose(); + public bool GetBoolean(int i) => _inner.GetBoolean(i); + public byte GetByte(int i) => _inner.GetByte(i); + public long GetBytes(int i, long fieldOffset, byte[]? buffer, int bufferoffset, int length) => _inner.GetBytes(i, fieldOffset, buffer, bufferoffset, length); + public char GetChar(int i) => _inner.GetChar(i); + public long GetChars(int i, long fieldoffset, char[]? buffer, int bufferoffset, int length) => _inner.GetChars(i, fieldoffset, buffer, bufferoffset, length); + public IDataReader GetData(int i) => _inner.GetData(i); + public string GetDataTypeName(int i) => _inner.GetDataTypeName(i); + public DateTime GetDateTime(int i) => _inner.GetDateTime(i); + public decimal GetDecimal(int i) => _inner.GetDecimal(i); + public double GetDouble(int i) => _inner.GetDouble(i); + public Type GetFieldType(int i) => _inner.GetFieldType(i); + public float GetFloat(int i) => _inner.GetFloat(i); + public Guid GetGuid(int i) => _inner.GetGuid(i); + public short GetInt16(int i) => _inner.GetInt16(i); + public int GetInt32(int i) => _inner.GetInt32(i); + public long GetInt64(int i) => _inner.GetInt64(i); + public string GetName(int i) => _inner.GetName(i); + public int GetOrdinal(string name) => _inner.GetOrdinal(name); + public DataTable GetSchemaTable() => _inner.GetSchemaTable()!; + public string GetString(int i) => _inner.GetString(i); + public object GetValue(int i) => _inner.GetValue(i); + public int GetValues(object[] values) => _inner.GetValues(values); + public bool IsDBNull(int i) => _inner.IsDBNull(i); + public bool NextResult() => _inner.NextResult(); +} +``` + +**Step 2: Update DatabaseExporter to use CountingDataReader** + +Update the ExportAsync method to wrap the reader: + +```csharp +// Replace this line: +await using var reader = await command.ExecuteReaderAsync(cancellationToken); + +// With: +await using var baseReader = await command.ExecuteReaderAsync(cancellationToken); +var reader = new CountingDataReader(baseReader); + +// And update the return to use reader.RowCount instead of 0 +``` + +**Step 3: Verify build** + +```bash +cd Tools/DbExporter && dotnet build +``` + +**Step 4: Commit** + +```bash +git add Tools/DbExporter/CountingDataReader.cs Tools/DbExporter/DatabaseExporter.cs +git commit -m "feat(DbExporter): add counting data reader for accurate row count" +``` + +--- + +### Task 6: Create ScadaBridge Definition Files + +**Files:** +- Create: `Tools/DbExporter/definitions/` directory with definition files + +**Step 1: Create definitions directory** + +```bash +mkdir -p Tools/DbExporter/definitions +``` + +**Step 2: Query ScadaBridge to list tables** + +First, we need to discover what tables exist. Run a quick query to list tables: + +```sql +SELECT TABLE_SCHEMA, TABLE_NAME +FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_TYPE = 'BASE TABLE' +ORDER BY TABLE_SCHEMA, TABLE_NAME +``` + +**Step 3: Create definition files for key tables** + +Create definition files based on discovered tables. Example for Config.ScadaClients: + +```json +{ + "providerType": "SqlServer", + "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", + "query": "SELECT * FROM Config.OpcUaClients", + "outputPath": "./output/opcua-clients.pb.zstd", + "compressionLevel": 10 +} +``` + +**Step 4: Commit** + +```bash +git add Tools/DbExporter/definitions/ +git commit -m "feat(DbExporter): add ScadaBridge definition files" +``` + +--- + +### Task 7: Test Export and Verify + +**Step 1: Build the tool** + +```bash +cd Tools/DbExporter && dotnet build +``` + +**Step 2: Run export for a small table first** + +```bash +dotnet run -- definitions/opcua-clients.json --verify +``` + +**Step 3: Run verify-full** + +```bash +dotnet run -- definitions/opcua-clients.json --verify-full +``` + +**Step 4: Test with larger tables if available** + +Run exports on additional definition files and verify they work correctly. + +**Step 5: Commit any fixes needed** + +```bash +git add -A +git commit -m "fix(DbExporter): address issues found during testing" +``` + +--- + +### Task 8: Final Cleanup and Documentation + +**Files:** +- Update: `Tools/DbExporter/README.md` (optional) + +**Step 1: Review all files for cleanup** + +- Remove any debug code +- Ensure consistent formatting +- Check for any TODO comments + +**Step 2: Final build and test** + +```bash +cd Tools/DbExporter && dotnet build +dotnet run -- --help +``` + +**Step 3: Commit** + +```bash +git add -A +git commit -m "chore(DbExporter): final cleanup" +``` diff --git a/Tools/DbExporter/.gitignore b/Tools/DbExporter/.gitignore new file mode 100644 index 0000000..a94f86a --- /dev/null +++ b/Tools/DbExporter/.gitignore @@ -0,0 +1,2 @@ +# Ignore exported output files +output/ diff --git a/Tools/DbExporter/DatabaseExporter.cs b/Tools/DbExporter/DatabaseExporter.cs index 40ef23d..5b197ea 100644 --- a/Tools/DbExporter/DatabaseExporter.cs +++ b/Tools/DbExporter/DatabaseExporter.cs @@ -29,30 +29,36 @@ public sealed class DatabaseExporter await using var baseReader = await command.ExecuteReaderAsync(cancellationToken); var reader = new CountingDataReader(baseReader); - long uncompressedSize = 0; + long uncompressedSize; + string hash; + int rowCount; // Use a counting stream wrapper to track uncompressed bytes - using var sha256 = SHA256.Create(); - await using var outputFile = new FileStream(definition.OutputPath, FileMode.Create, FileAccess.Write, FileShare.None, 256 * 1024); - await using var compressStream = new CompressionStream(outputFile, definition.CompressionLevel); - await using var countingStream = new CountingStream(compressStream); - await using var hashStream = new CryptoStream(countingStream, sha256, CryptoStreamMode.Write); + using (var sha256 = SHA256.Create()) + { + await using var outputFile = new FileStream(definition.OutputPath, FileMode.Create, FileAccess.Write, FileShare.None, 256 * 1024); + await using var compressStream = new CompressionStream(outputFile, definition.CompressionLevel); + await using var countingStream = new CountingStream(compressStream); + await using var hashStream = new CryptoStream(countingStream, sha256, CryptoStreamMode.Write); - // Serialize to protobuf - DataSerializer.Serialize(hashStream, reader); + // Serialize to protobuf + DataSerializer.Serialize(hashStream, reader); - hashStream.FlushFinalBlock(); - uncompressedSize = countingStream.BytesWritten; + hashStream.FlushFinalBlock(); + uncompressedSize = countingStream.BytesWritten; + rowCount = reader.RowCount; - var hash = Convert.ToHexString(sha256.Hash!).ToLowerInvariant(); + hash = Convert.ToHexString(sha256.Hash!).ToLowerInvariant(); + } // All streams disposed here, file fully written // Write sidecar hash file var hashFilePath = definition.OutputPath + ".sha256"; await File.WriteAllTextAsync(hashFilePath, hash, cancellationToken); + // Read file size after streams are closed var compressedSize = new FileInfo(definition.OutputPath).Length; - return new ExportResult(reader.RowCount, uncompressedSize, compressedSize, hash); + return new ExportResult(rowCount, uncompressedSize, compressedSize, hash); } private static DbConnection CreateConnection(string providerType, string connectionString) diff --git a/Tools/DbExporter/definitions/email-providers.json b/Tools/DbExporter/definitions/email-providers.json deleted file mode 100644 index 75effd5..0000000 --- a/Tools/DbExporter/definitions/email-providers.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Config.EmailProviders", - "outputPath": "./output/email-providers.pb.zstd", - "compressionLevel": 10 -} diff --git a/Tools/DbExporter/definitions/tags.json b/Tools/DbExporter/definitions/example-sys-tables.json similarity index 67% rename from Tools/DbExporter/definitions/tags.json rename to Tools/DbExporter/definitions/example-sys-tables.json index 00bcdf3..b19d69e 100644 --- a/Tools/DbExporter/definitions/tags.json +++ b/Tools/DbExporter/definitions/example-sys-tables.json @@ -1,7 +1,7 @@ { "providerType": "SqlServer", "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Config.Tags", - "outputPath": "./output/tags.pb.zstd", + "query": "SELECT * FROM sys.tables", + "outputPath": "./output/sys-tables.pb.zstd", "compressionLevel": 10 } diff --git a/Tools/DbExporter/definitions/example-tables.json b/Tools/DbExporter/definitions/example-tables.json new file mode 100644 index 0000000..3cab9b4 --- /dev/null +++ b/Tools/DbExporter/definitions/example-tables.json @@ -0,0 +1,7 @@ +{ + "providerType": "SqlServer", + "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", + "query": "SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_SCHEMA, TABLE_NAME", + "outputPath": "./output/test-schema.pb.zstd", + "compressionLevel": 10 +} diff --git a/Tools/DbExporter/definitions/lmx-clients.json b/Tools/DbExporter/definitions/lmx-clients.json deleted file mode 100644 index 9a0f68a..0000000 --- a/Tools/DbExporter/definitions/lmx-clients.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Config.LmxClients", - "outputPath": "./output/lmx-clients.pb.zstd", - "compressionLevel": 10 -} diff --git a/Tools/DbExporter/definitions/lookup-data-types.json b/Tools/DbExporter/definitions/lookup-data-types.json deleted file mode 100644 index 4b3f097..0000000 --- a/Tools/DbExporter/definitions/lookup-data-types.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Lookup.DataTypes", - "outputPath": "./output/lookup-data-types.pb.zstd", - "compressionLevel": 10 -} diff --git a/Tools/DbExporter/definitions/lookup-scada-client-types.json b/Tools/DbExporter/definitions/lookup-scada-client-types.json deleted file mode 100644 index 3c30b3a..0000000 --- a/Tools/DbExporter/definitions/lookup-scada-client-types.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Lookup.ScadaClientTypes", - "outputPath": "./output/lookup-scada-client-types.pb.zstd", - "compressionLevel": 10 -} diff --git a/Tools/DbExporter/definitions/lookup-script-types.json b/Tools/DbExporter/definitions/lookup-script-types.json deleted file mode 100644 index e9ba9dd..0000000 --- a/Tools/DbExporter/definitions/lookup-script-types.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Lookup.ScriptTypes", - "outputPath": "./output/lookup-script-types.pb.zstd", - "compressionLevel": 10 -} diff --git a/Tools/DbExporter/definitions/lookup-trigger-types.json b/Tools/DbExporter/definitions/lookup-trigger-types.json deleted file mode 100644 index e2c94e9..0000000 --- a/Tools/DbExporter/definitions/lookup-trigger-types.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Lookup.TriggerTypes", - "outputPath": "./output/lookup-trigger-types.pb.zstd", - "compressionLevel": 10 -} diff --git a/Tools/DbExporter/definitions/opcua-clients.json b/Tools/DbExporter/definitions/opcua-clients.json deleted file mode 100644 index 224d071..0000000 --- a/Tools/DbExporter/definitions/opcua-clients.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Config.OpcUaClients", - "outputPath": "./output/opcua-clients.pb.zstd", - "compressionLevel": 10 -} diff --git a/Tools/DbExporter/definitions/sms-providers.json b/Tools/DbExporter/definitions/sms-providers.json deleted file mode 100644 index d1fd293..0000000 --- a/Tools/DbExporter/definitions/sms-providers.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Config.SmsProviders", - "outputPath": "./output/sms-providers.pb.zstd", - "compressionLevel": 10 -} diff --git a/Tools/DbExporter/definitions/teams-providers.json b/Tools/DbExporter/definitions/teams-providers.json deleted file mode 100644 index 266d9e2..0000000 --- a/Tools/DbExporter/definitions/teams-providers.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Config.TeamsProviders", - "outputPath": "./output/teams-providers.pb.zstd", - "compressionLevel": 10 -} diff --git a/Tools/DbExporter/definitions/template-scripts.json b/Tools/DbExporter/definitions/template-scripts.json deleted file mode 100644 index 291ec04..0000000 --- a/Tools/DbExporter/definitions/template-scripts.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "providerType": "SqlServer", - "connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;", - "query": "SELECT * FROM Config.TemplateScripts", - "outputPath": "./output/template-scripts.pb.zstd", - "compressionLevel": 10 -}