Joseph Doherty
e5fe2f06e9
Add ConfigurationValidationRunner with IConfigurationValidator interface for validating required settings at startup. Includes SecureStore and LDAP validators. Expand ConfigManager with pipeline editing UI, dialogs, and step editors. Update documentation with config validation guidance.
119 lines
3.0 KiB
Bash
Executable File
119 lines
3.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# Export all tables (except Search*) from ScopingTool QA database
|
|
# Output: /Volumes/DOCK_NVME/CACHED_DB_FILES
|
|
# Runs 8 tables in parallel
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
OUTPUT_DIR="/Volumes/DOCK_NVME/CACHED_DB_FILES"
|
|
DEFINITIONS_DIR="$SCRIPT_DIR/definitions/export-batch"
|
|
CONNECTION_STRING="Server=JDESCP-SQL-VQ01.zmr.zimmer.com;Database=ScopingTool;User Id=ScopingTool;Password=Tool#qascope;TrustServerCertificate=true;"
|
|
PARALLEL_JOBS=8
|
|
|
|
# Tables to export (excluding Search* tables)
|
|
TABLES=(
|
|
"Branch"
|
|
"DataUpdate"
|
|
"FunctionCode"
|
|
"Item"
|
|
"JdeUser"
|
|
"Lot"
|
|
"LotUsage_Curr"
|
|
"LotUsage_Hist"
|
|
"MisData"
|
|
"OrgHierarchy"
|
|
"ProfitCenter"
|
|
"RouteMaster"
|
|
"StatusCode"
|
|
"WorkCenter"
|
|
"WorkOrder_Curr"
|
|
"WorkOrder_Hist"
|
|
"WorkOrderComponent_Curr"
|
|
"WorkOrderComponent_Hist"
|
|
"WorkOrderRouting"
|
|
"WorkOrderStep_Curr"
|
|
"WorkOrderStep_Hist"
|
|
"WorkOrderTime_Curr"
|
|
"WorkOrderTime_Hist"
|
|
)
|
|
|
|
echo "=================================================="
|
|
echo "DbExporter - Batch Export (Parallel)"
|
|
echo "=================================================="
|
|
echo "Database: ScopingTool (QA)"
|
|
echo "Output: $OUTPUT_DIR"
|
|
echo "Tables: ${#TABLES[@]}"
|
|
echo "Parallel: $PARALLEL_JOBS at a time"
|
|
echo "=================================================="
|
|
echo ""
|
|
|
|
# Create directories
|
|
mkdir -p "$DEFINITIONS_DIR"
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
# Build the project first
|
|
echo "Building DbExporter..."
|
|
cd "$SCRIPT_DIR"
|
|
dotnet build -c Release --nologo -v q
|
|
echo ""
|
|
|
|
EXPORTER="$SCRIPT_DIR/bin/Release/net10.0/DbExporter"
|
|
|
|
# Function to export a single table
|
|
export_table() {
|
|
local TABLE="$1"
|
|
local EXPORTER="$2"
|
|
local DEFINITIONS_DIR="$3"
|
|
local OUTPUT_DIR="$4"
|
|
local CONNECTION_STRING="$5"
|
|
|
|
TABLE_LOWER=$(echo "$TABLE" | tr '[:upper:]' '[:lower:]')
|
|
DEFINITION_FILE="$DEFINITIONS_DIR/${TABLE_LOWER}.json"
|
|
OUTPUT_FILE="$OUTPUT_DIR/${TABLE_LOWER}.pb.zstd"
|
|
|
|
# Create definition file
|
|
cat > "$DEFINITION_FILE" << EOF
|
|
{
|
|
"providerType": "SqlServer",
|
|
"connectionString": "$CONNECTION_STRING",
|
|
"query": "SELECT * FROM [$TABLE]",
|
|
"outputPath": "$OUTPUT_FILE",
|
|
"compressionLevel": 10
|
|
}
|
|
EOF
|
|
|
|
# Run export
|
|
echo "[$TABLE] Starting..."
|
|
if $EXPORTER "$DEFINITION_FILE" 2>&1 | sed "s/^/[$TABLE] /"; then
|
|
echo "[$TABLE] Done"
|
|
else
|
|
echo "[$TABLE] FAILED"
|
|
fi
|
|
}
|
|
|
|
export -f export_table
|
|
|
|
# Track progress
|
|
TOTAL=${#TABLES[@]}
|
|
|
|
echo "Starting parallel export of $TOTAL tables ($PARALLEL_JOBS at a time)..."
|
|
echo ""
|
|
|
|
# Run exports in parallel, 8 at a time
|
|
printf '%s\n' "${TABLES[@]}" | xargs -P $PARALLEL_JOBS -I {} bash -c 'export_table "$@"' _ {} "$EXPORTER" "$DEFINITIONS_DIR" "$OUTPUT_DIR" "$CONNECTION_STRING"
|
|
|
|
echo ""
|
|
echo "=================================================="
|
|
echo "Export Complete"
|
|
echo "=================================================="
|
|
|
|
# Show output files
|
|
echo ""
|
|
echo "Output files:"
|
|
ls -lhS "$OUTPUT_DIR"/*.pb.zstd 2>/dev/null | awk '{printf " %-50s %8s\n", $9, $5}'
|
|
|
|
echo ""
|
|
echo "Total size:"
|
|
du -sh "$OUTPUT_DIR"
|