dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(DbExporter): fix compressed size calculation and clean up

Browse Source 
- Move file size read after streams are disposed to get accurate compressed size
- Clean up definition files to use working example queries
- Add .gitignore for output directory
This commit is contained in:
Joseph Doherty
2026-01-06 17:06:16 -05:00
parent f61a64b059
commit d2136cacf7
16 changed files with 795 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tools/DbExporter/.gitignore
+2
View File
@@ -0,0 +1,2 @@
# Ignore exported output files
output/
Tools/DbExporter/DatabaseExporter.cs
+18 -12
View File
@@ -29,30 +29,36 @@ public sealed class DatabaseExporter
await using var baseReader = await command.ExecuteReaderAsync(cancellationToken);
var reader = new CountingDataReader(baseReader);
long uncompressedSize = 0;
long uncompressedSize;
string hash;
int rowCount;
// Use a counting stream wrapper to track uncompressed bytes
using var sha256 = SHA256.Create();
await using var outputFile = new FileStream(definition.OutputPath, FileMode.Create, FileAccess.Write, FileShare.None, 256 * 1024);
await using var compressStream = new CompressionStream(outputFile, definition.CompressionLevel);
await using var countingStream = new CountingStream(compressStream);
await using var hashStream = new CryptoStream(countingStream, sha256, CryptoStreamMode.Write);
using (var sha256 = SHA256.Create())
{
await using var outputFile = new FileStream(definition.OutputPath, FileMode.Create, FileAccess.Write, FileShare.None, 256 * 1024);
await using var compressStream = new CompressionStream(outputFile, definition.CompressionLevel);
await using var countingStream = new CountingStream(compressStream);
await using var hashStream = new CryptoStream(countingStream, sha256, CryptoStreamMode.Write);
// Serialize to protobuf
DataSerializer.Serialize(hashStream, reader);
// Serialize to protobuf
DataSerializer.Serialize(hashStream, reader);
hashStream.FlushFinalBlock();
uncompressedSize = countingStream.BytesWritten;
hashStream.FlushFinalBlock();
uncompressedSize = countingStream.BytesWritten;
rowCount = reader.RowCount;
var hash = Convert.ToHexString(sha256.Hash!).ToLowerInvariant();
hash = Convert.ToHexString(sha256.Hash!).ToLowerInvariant();
} // All streams disposed here, file fully written
// Write sidecar hash file
var hashFilePath = definition.OutputPath + ".sha256";
await File.WriteAllTextAsync(hashFilePath, hash, cancellationToken);
// Read file size after streams are closed
var compressedSize = new FileInfo(definition.OutputPath).Length;
return new ExportResult(reader.RowCount, uncompressedSize, compressedSize, hash);
return new ExportResult(rowCount, uncompressedSize, compressedSize, hash);
}
private static DbConnection CreateConnection(string providerType, string connectionString)
Tools/DbExporter/definitions/email-providers.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Config.EmailProviders",
"outputPath": "./output/email-providers.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/tags.json → Tools/DbExporter/definitions/example-sys-tables.json
+2 -2
View File
@@ -1,7 +1,7 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Config.Tags",
"outputPath": "./output/tags.pb.zstd",
"query": "SELECT * FROM sys.tables",
"outputPath": "./output/sys-tables.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/example-tables.json
+7
View File
@@ -0,0 +1,7 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_SCHEMA, TABLE_NAME",
"outputPath": "./output/test-schema.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/lmx-clients.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Config.LmxClients",
"outputPath": "./output/lmx-clients.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/lookup-data-types.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Lookup.DataTypes",
"outputPath": "./output/lookup-data-types.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/lookup-scada-client-types.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Lookup.ScadaClientTypes",
"outputPath": "./output/lookup-scada-client-types.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/lookup-script-types.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Lookup.ScriptTypes",
"outputPath": "./output/lookup-script-types.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/lookup-trigger-types.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Lookup.TriggerTypes",
"outputPath": "./output/lookup-trigger-types.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/opcua-clients.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Config.OpcUaClients",
"outputPath": "./output/opcua-clients.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/sms-providers.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Config.SmsProviders",
"outputPath": "./output/sms-providers.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/teams-providers.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Config.TeamsProviders",
"outputPath": "./output/teams-providers.pb.zstd",
"compressionLevel": 10
}
Tools/DbExporter/definitions/template-scripts.json
-7
View File
@@ -1,7 +0,0 @@
{
"providerType": "SqlServer",
"connectionString": "Server=10.100.0.35;Database=ScadaBridge_Test;User Id=sa;Password=ScadaBridge2024;TrustServerCertificate=true;",
"query": "SELECT * FROM Config.TemplateScripts",
"outputPath": "./output/template-scripts.pb.zstd",
"compressionLevel": 10
}