Initial commit: JDE Scoping Tool migration project
Set up repository with legacy .NET Framework 4.8 source (OLD/), new .NET 10 Blazor solution (NEW/), OpenSpec specifications, documentation, and project configuration.
This commit is contained in:
Joseph Doherty
Executable
+75
@@ -0,0 +1,75 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace WorkerService.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// Action (void function) JSON converter
|
||||
/// </summary>
|
||||
public class ActionConverter : JsonConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes the JSON representation of the object
|
||||
/// </summary>
|
||||
/// <param name="writer">The JsonWriter to write to</param>
|
||||
/// <param name="value">The value to write</param>
|
||||
/// <param name="serializer">The calling serializer</param>
|
||||
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
|
||||
{
|
||||
if (value is Action action)
|
||||
{
|
||||
writer.WriteValue($"{action.Method.DeclaringType}.{action.Method.Name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Reads the JSON representation of the object.</summary>
|
||||
/// <param name="reader">The <see cref="T:Newtonsoft.Json.JsonReader" /> to read from.</param>
|
||||
/// <param name="objectType">Type of the object.</param>
|
||||
/// <param name="existingValue">The existing value of object being read.</param>
|
||||
/// <param name="serializer">The calling serializer.</param>
|
||||
/// <returns>The object value.</returns>
|
||||
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
|
||||
{
|
||||
//Get the method's full path
|
||||
string fullMethodPath = (string)reader.Value;
|
||||
if (string.IsNullOrEmpty(fullMethodPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
//Extract class and method's names
|
||||
string className = fullMethodPath.Substring(0, fullMethodPath.LastIndexOf(".", StringComparison.Ordinal));
|
||||
string methodName = fullMethodPath.Substring(fullMethodPath.LastIndexOf(".", StringComparison.Ordinal) + 1);
|
||||
|
||||
//Get the class type
|
||||
Type classType = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).FirstOrDefault(t => string.Equals(t.FullName, className, StringComparison.CurrentCultureIgnoreCase));
|
||||
if (classType == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
//Get the function's method info
|
||||
MethodInfo methodInfo = classType.GetMethod(methodName);
|
||||
if (methodInfo == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return (Action)methodInfo.CreateDelegate(typeof(Action));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether this instance can convert the specified object type
|
||||
/// </summary>
|
||||
/// <param name="objectType">Type of the object</param>
|
||||
/// <returns>
|
||||
/// Whether or not this instance can convert ot the specified object type
|
||||
/// </returns>
|
||||
public override bool CanConvert(Type objectType)
|
||||
{
|
||||
return objectType == typeof(Action);
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+499
@@ -0,0 +1,499 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Drawing;
|
||||
using System.IO;
|
||||
using DataModel.Models;
|
||||
using OfficeOpenXml;
|
||||
using OfficeOpenXml.Style;
|
||||
using OfficeOpenXml.Table;
|
||||
using WorkerService.Helpers;
|
||||
using WorkerService.Models.Reporting;
|
||||
|
||||
namespace WorkerService.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// Search results report writer
|
||||
/// </summary>
|
||||
public class ExcelWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Standard date format
|
||||
/// </summary>
|
||||
private const string DATE_FORMAT = "{0:MM/dd/yyyy}";
|
||||
|
||||
/// <summary>
|
||||
/// Standard timestamp format
|
||||
/// </summary>
|
||||
private const string TIMESTAMP_FORMAT = "[$-409]m/d/yy h:mm AM/PM;@";
|
||||
|
||||
/// <summary>
|
||||
/// Override width for cells with wrapped data
|
||||
/// </summary>
|
||||
private const double WRAPPED_CELL_WIDTH = 65;
|
||||
|
||||
/// <summary>
|
||||
/// Generates Excel report for the given search criteria and results
|
||||
/// </summary>
|
||||
/// <param name="searchModel">Search model to generate report for</param>
|
||||
/// <returns>Excel report file contents</returns>
|
||||
public static byte[] Generate(SearchModel searchModel)
|
||||
{
|
||||
byte[] data;
|
||||
|
||||
using (ExcelPackage package = new ExcelPackage())
|
||||
{
|
||||
//Write search criteria tab
|
||||
WriteCriteria(package, searchModel);
|
||||
|
||||
//Write results tab
|
||||
package.Workbook.LoadTab(searchModel.Results);
|
||||
|
||||
//Write MIS tab
|
||||
if(searchModel.ExtractMisData)
|
||||
{
|
||||
package.Workbook.LoadTab(searchModel.MisResults);
|
||||
}
|
||||
|
||||
//Write mismatch tab
|
||||
if (searchModel.ExtractMisData)
|
||||
{
|
||||
package.Workbook.LoadTab(searchModel.MisNonMatchResults);
|
||||
}
|
||||
|
||||
//Save workbook to array
|
||||
using (MemoryStream memoryStream = new MemoryStream())
|
||||
{
|
||||
package.SaveAs(memoryStream);
|
||||
memoryStream.Position = 0;
|
||||
data = memoryStream.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes search criteria to a tab in the worksheet
|
||||
/// </summary>
|
||||
/// <param name="package">Excel package to write to</param>
|
||||
/// <param name="searchModel">Search model to generate report for</param>
|
||||
private static void WriteCriteria(ExcelPackage package, SearchModel searchModel)
|
||||
{
|
||||
//Create table in workbook to hold criteria
|
||||
ExcelWorksheet criteriaTab = package.Workbook.Worksheets.Add("Search Criteria");
|
||||
|
||||
int row = 1;
|
||||
|
||||
//Write name and revision number
|
||||
ApplyHeaderFormat(criteriaTab.Cells[row, 1], "Search Name");
|
||||
criteriaTab.Cells[row, 2].Value = searchModel.Name;
|
||||
|
||||
ApplyHeaderFormat(criteriaTab.Cells[++row, 1], "User Name");
|
||||
criteriaTab.Cells[row, 2].Value = searchModel.UserName;
|
||||
|
||||
//Skip row
|
||||
row++;
|
||||
|
||||
//Write timestamps
|
||||
ApplyHeaderFormat(criteriaTab.Cells[++row, 1], "Submit timestamp");
|
||||
criteriaTab.Cells[row, 2].Value = $"{searchModel.SubmitDT:MMM dd, yyyy hh:mm:ss tt} EST";
|
||||
|
||||
ApplyHeaderFormat(criteriaTab.Cells[++row, 1], "Start timestamp");
|
||||
criteriaTab.Cells[row, 2].Value = $"{searchModel.StartDT:MMM dd, yyyy hh:mm:ss tt} EST";
|
||||
|
||||
ApplyHeaderFormat(criteriaTab.Cells[++row, 1], "Completed timestamp");
|
||||
criteriaTab.Cells[row, 2].Value = $"{searchModel.EndDT:MMM dd, yyyy hh:mm:ss tt} EST";
|
||||
|
||||
//Skip row
|
||||
row++;
|
||||
|
||||
/*
|
||||
* Write min/max times
|
||||
*/
|
||||
ExcelTable timespanFilterTable = criteriaTab.Cells[++row, 1].LoadTable(
|
||||
new List<TimespanFilter>()
|
||||
{
|
||||
new TimespanFilter()
|
||||
{
|
||||
MinimumDT = searchModel.MinimumDT, MaximumDT = searchModel.MaximumDT
|
||||
}
|
||||
}
|
||||
);
|
||||
row = timespanFilterTable.Address.End.Row + 3;
|
||||
|
||||
/*
|
||||
* Write lot numbers
|
||||
*/
|
||||
ExcelTable workOrderFilterTable = criteriaTab.Cells[row, 1].LoadTable(searchModel.WorkOrderFilter);
|
||||
row = workOrderFilterTable.Address.End.Row +3;
|
||||
|
||||
/*
|
||||
* Write item numbers
|
||||
*/
|
||||
ExcelTable itemNumberFilterTable = criteriaTab.Cells[row, 1].LoadTable(searchModel.ItemNumberFilter);
|
||||
row = itemNumberFilterTable.Address.End.Row +3;
|
||||
|
||||
/*
|
||||
* Write profit centers
|
||||
*/
|
||||
ExcelTable profitCenterFilterTable = criteriaTab.Cells[row, 1].LoadTable(searchModel.ProfitCenterFilter);
|
||||
row = profitCenterFilterTable.Address.End.Row + 3;
|
||||
|
||||
/*
|
||||
* Write work centers
|
||||
*/
|
||||
ExcelTable workCenterFilterTable = criteriaTab.Cells[row, 1].LoadTable(searchModel.WorkCenterFilter);
|
||||
row = workCenterFilterTable.Address.End.Row + 3;
|
||||
|
||||
/*
|
||||
* Write component lot numbers
|
||||
*/
|
||||
ExcelTable componentLotFilterTable = criteriaTab.Cells[row, 1].LoadTable(searchModel.ComponentLotFilter);
|
||||
row = componentLotFilterTable.Address.End.Row + 3;
|
||||
|
||||
/*
|
||||
* Write operators
|
||||
*/
|
||||
ExcelTable operatorFilterTable = criteriaTab.Cells[row, 1].LoadTable(searchModel.OperatorFilter);
|
||||
row = operatorFilterTable.Address.End.Row + 3;
|
||||
|
||||
/*
|
||||
* Write item/operation/mis
|
||||
*/
|
||||
ExcelTable itemOperationMisFilterTable = criteriaTab.Cells[row, 1].LoadTable(searchModel.ItemOperationMisFilter);
|
||||
row = itemOperationMisFilterTable.Address.End.Row + 3;
|
||||
|
||||
/*
|
||||
* Write extract MIS data option
|
||||
*/
|
||||
ApplyHeaderFormat(criteriaTab.Cells[row, 1, row, 2], "Extract MIS data?", true);
|
||||
criteriaTab.Cells[++row, 1].Value = searchModel.ExtractMisData ? "YES" : "NO";
|
||||
|
||||
//Auto-fit columns
|
||||
for (int column = 1; column <= 4; column++)
|
||||
{
|
||||
criteriaTab.Column(column).AutoFit();
|
||||
criteriaTab.Column(column).Width = criteriaTab.Column(column).Width * 1.15;
|
||||
}
|
||||
|
||||
//Set worksheet/tab as protected
|
||||
criteriaTab.Protection.SetPassword("JDE_SCOPING_TOOL_PASS");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes search results to a tab in the worksheet
|
||||
/// </summary>
|
||||
/// <param name="package">Excel package to write to</param>
|
||||
/// <param name="search">Search to extract criteria from</param>
|
||||
/// <param name="results">Search results to write</param>
|
||||
private static void WriteResults(ExcelPackage package, Search search, List<SearchResult> results)
|
||||
{
|
||||
//Create tab in workbook to hold results
|
||||
ExcelWorksheet resultsTab = package.Workbook.Worksheets.Add("Search Results");
|
||||
|
||||
int row = 1;
|
||||
int col = 1;
|
||||
|
||||
//Write header
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Work Order Number");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Work Order Branch Code");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Lot Number");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Item Number");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Planning Family");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Order Quantity");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Held Quantity");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Scrapped Quantity");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Shipped Quantity");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Operation Step Branch Code");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Operation Step");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Operation Step Description");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Function Operation Description");
|
||||
|
||||
resultsTab.Column(col).Style.Numberformat.Format = TIMESTAMP_FORMAT;
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Operation Step Update Timestamp");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Status Code");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Status Description");
|
||||
|
||||
resultsTab.Column(col).Style.Numberformat.Format = TIMESTAMP_FORMAT;
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Status Update Timestamp");
|
||||
ApplyHeaderFormat(resultsTab.Cells[row, col++], "Inclusion Reason");
|
||||
|
||||
|
||||
//Write data
|
||||
foreach (SearchResult searchResult in results)
|
||||
{
|
||||
row++;
|
||||
col = 1;
|
||||
|
||||
resultsTab.Cells[row, col++].Value = searchResult.WorkOrderNumber;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.WorkOrderBranchCode;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.LotNumber;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.ItemNumber;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.PlanningFamily;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.OrderQuantity;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.HeldQuantity;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.ScrappedQuantity;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.ShippedQuantity;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.StepBranchCode;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.StepNumber;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.StepDescription;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.FunctionOperationDescription;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.StepUpdateDT;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.StatusCode;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.StatusDescription;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.StatusUpdateDT;
|
||||
resultsTab.Cells[row, col++].Value = searchResult.InclusionReason;
|
||||
}
|
||||
|
||||
//Auto-fit columns
|
||||
for (int column = 1; column <= resultsTab.Dimension.Columns; column++)
|
||||
{
|
||||
resultsTab.Column(column).AutoFit();
|
||||
resultsTab.Column(column).Width = resultsTab.Column(column).Width * 1.3;
|
||||
}
|
||||
|
||||
//Get protected / unprotected ranges for editing
|
||||
ExcelRange protectedRange = resultsTab.Cells[1, 1, row, resultsTab.Dimension.Columns];
|
||||
ExcelRange unprotectedRange = resultsTab.Cells[1, resultsTab.Dimension.Columns + 1, row + 1000, resultsTab.Dimension.Columns + 1000];
|
||||
|
||||
//Format as table
|
||||
ExcelTable table = resultsTab.Tables.Add(protectedRange, "Search_Results");
|
||||
table.ShowTotal = false;
|
||||
table.TableStyle = TableStyles.Medium1;
|
||||
|
||||
//Write-protect range
|
||||
resultsTab.Protection.IsProtected = true;
|
||||
resultsTab.ProtectedRanges.Add("Editable", unprotectedRange);
|
||||
resultsTab.Protection.AllowDeleteColumns = true;
|
||||
resultsTab.Protection.AllowDeleteRows = false;
|
||||
resultsTab.Protection.AllowAutoFilter = true;
|
||||
resultsTab.Protection.AllowAutoFilter = true;
|
||||
resultsTab.Protection.AllowFormatCells = true;
|
||||
resultsTab.Protection.AllowFormatColumns = true;
|
||||
resultsTab.Protection.AllowFormatRows = true;
|
||||
resultsTab.Protection.AllowSelectLockedCells = true;
|
||||
resultsTab.Protection.AllowSelectUnlockedCells = true;
|
||||
resultsTab.Protection.AllowEditObject = true;
|
||||
resultsTab.Protection.AllowSort = true;
|
||||
resultsTab.Protection.SetPassword("JDESCOPINGTOOL");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes intermediate MIS search results to a tab in the worksheet
|
||||
/// </summary>
|
||||
/// <param name="package">Excel package to write to</param>
|
||||
/// <param name="misInfo">Intermediate MIS search results to write</param>
|
||||
private static void WriteMisInfo(ExcelPackage package, List<MisSearchResult> misInfo)
|
||||
{
|
||||
if (misInfo == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
//Create tab in workbook to hold results
|
||||
ExcelWorksheet misInfoTab = package.Workbook.Worksheets.Add("MIS Info");
|
||||
|
||||
int row = 1;
|
||||
int col = 1;
|
||||
|
||||
//Write header
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Item Number");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Item Description");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "MIS Job Step Sequence Number");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "MIS Number");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "MIS Revision");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "MIS Release Status");
|
||||
|
||||
misInfoTab.Column(col).Style.Numberformat.Format = TIMESTAMP_FORMAT;
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "MIS Release Date");
|
||||
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Branch Code");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Job Step Sequence Number");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Matched Sequence Number");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Matched to F3112Z1?");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Matched to F3003?");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Function Operation Description");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Char Number");
|
||||
|
||||
misInfoTab.Column(col).Style.WrapText = true;
|
||||
misInfoTab.Column(col).Width = 65;
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Test Description");
|
||||
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Sampling Type");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Sampling Value");
|
||||
|
||||
misInfoTab.Column(col).Style.WrapText = true;
|
||||
misInfoTab.Column(col).Width = WRAPPED_CELL_WIDTH;
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Tools & Gauges");
|
||||
|
||||
misInfoTab.Column(col).Style.WrapText = true;
|
||||
misInfoTab.Column(col).Width = WRAPPED_CELL_WIDTH;
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Work Instructions");
|
||||
|
||||
//Write data
|
||||
foreach (MisSearchResult misData in misInfo)
|
||||
{
|
||||
row++;
|
||||
col = 1;
|
||||
|
||||
misInfoTab.Cells[row, col++].Value = misData.ItemNumber;
|
||||
misInfoTab.Cells[row, col++].Value = misData.ItemDescription;
|
||||
misInfoTab.Cells[row, col++].Value = misData.SequenceNumber;
|
||||
misInfoTab.Cells[row, col++].Value = misData.MisNumber;
|
||||
misInfoTab.Cells[row, col++].Value = misData.RevID;
|
||||
misInfoTab.Cells[row, col++].Value = misData.Status;
|
||||
misInfoTab.Cells[row, col++].Value = misData.ReleaseDate;
|
||||
misInfoTab.Cells[row, col++].Value = misData.BranchCode;
|
||||
misInfoTab.Cells[row, col++].Value = misData.JobStepSequenceNumber;
|
||||
misInfoTab.Cells[row, col++].Value = misData.MatchedSequenceNumber;
|
||||
misInfoTab.Cells[row, col++].Value = misData.RoutingMatch;
|
||||
misInfoTab.Cells[row, col++].Value = misData.MasterMatch;
|
||||
misInfoTab.Cells[row, col++].Value = misData.FunctionOperationDescription;
|
||||
misInfoTab.Cells[row, col++].Value = misData.CharNumber;
|
||||
misInfoTab.Cells[row, col++].Value = misData.TestDescription;
|
||||
misInfoTab.Cells[row, col++].Value = misData.SamplingType;
|
||||
misInfoTab.Cells[row, col++].Value = misData.SamplingValue;
|
||||
misInfoTab.Cells[row, col++].Value = misData.ToolsGauges;
|
||||
misInfoTab.Cells[row, col++].Value = misData.WorkInstructions;
|
||||
}
|
||||
|
||||
//Auto-fit columns
|
||||
for (int column = 1; column <= misInfoTab.Dimension.Columns; column++)
|
||||
{
|
||||
if (misInfoTab.Column(column).Width == WRAPPED_CELL_WIDTH)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
misInfoTab.Column(column).AutoFit();
|
||||
misInfoTab.Column(column).Width = misInfoTab.Column(column).Width * 1.3;
|
||||
}
|
||||
|
||||
//Get protected / unprotected ranges for editing
|
||||
ExcelRange protectedRange = misInfoTab.Cells[1, 1, row, misInfoTab.Dimension.Columns];
|
||||
ExcelRange unprotectedRange = misInfoTab.Cells[1, misInfoTab.Dimension.Columns + 1, row + 1000, misInfoTab.Dimension.Columns + 1000];
|
||||
|
||||
//Format as table
|
||||
ExcelTable table = misInfoTab.Tables.Add(protectedRange, "MIS_Info");
|
||||
table.ShowTotal = false;
|
||||
table.TableStyle = TableStyles.Medium1;
|
||||
|
||||
//Write-protect data
|
||||
ApplySecurity(misInfoTab);
|
||||
misInfoTab.ProtectedRanges.Add("Editable", unprotectedRange);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes master router mismatches to a tab in the worksheet
|
||||
/// </summary>
|
||||
/// <param name="package">Excel package to write to</param>
|
||||
/// <param name="misMatches">Master router mismatches to write</param>
|
||||
private static void WriteRouterMismatches(ExcelPackage package, List<MisNonMatchSearchResult> misMatches)
|
||||
{
|
||||
if (misMatches == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
//Create tab in workbook to hold results
|
||||
ExcelWorksheet misInfoTab = package.Workbook.Worksheets.Add("Investigation");
|
||||
|
||||
int row = 1;
|
||||
int col = 1;
|
||||
|
||||
//Write header
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Work Center Code");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Work Order Number");
|
||||
|
||||
misInfoTab.Column(col).Style.Numberformat.Format = "m/d/yyyy";
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Work Order Start Date");
|
||||
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Job Step Number");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Function Operation Description");
|
||||
|
||||
misInfoTab.Column(col).Style.Numberformat.Format = "m/d/yyyy";
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Job Step End Date");
|
||||
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Function Code");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Item Number");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Item Description");
|
||||
ApplyHeaderFormat(misInfoTab.Cells[row, col++], "Routing Type");
|
||||
|
||||
//Write data
|
||||
foreach (MisNonMatchSearchResult misMatch in misMatches)
|
||||
{
|
||||
row++;
|
||||
col = 1;
|
||||
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.WorkCenterCode;
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.WorkOrderNumber;
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.WorkOrderStartDate;
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.JobStepNumber;
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.JobStepDescription;
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.JobStepEndDate;
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.FunctionCode;
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.ItemNumber;
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.ItemDescription;
|
||||
misInfoTab.Cells[row, col++].Value = misMatch.RoutingType;
|
||||
}
|
||||
|
||||
//Auto-fit columns
|
||||
for (int column = 1; column <= misInfoTab.Dimension.Columns; column++)
|
||||
{
|
||||
misInfoTab.Column(column).AutoFit();
|
||||
misInfoTab.Column(column).Width = misInfoTab.Column(column).Width * 1.3;
|
||||
}
|
||||
|
||||
//Get protected / unprotected ranges for editing
|
||||
ExcelRange protectedRange = misInfoTab.Cells[1, 1, row, misInfoTab.Dimension.Columns];
|
||||
ExcelRange unprotectedRange = misInfoTab.Cells[1, misInfoTab.Dimension.Columns + 1, row + 1000, misInfoTab.Dimension.Columns + 1000];
|
||||
|
||||
//Format as table
|
||||
ExcelTable table = misInfoTab.Tables.Add(protectedRange, "Investigation");
|
||||
table.ShowTotal = false;
|
||||
table.TableStyle = TableStyles.Medium1;
|
||||
|
||||
//Write-protect data
|
||||
ApplySecurity(misInfoTab);
|
||||
misInfoTab.ProtectedRanges.Add("Editable", unprotectedRange);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies header formatting to the range of cells
|
||||
/// </summary>
|
||||
/// <param name="range">Range of cells to format</param>
|
||||
/// <param name="text">Text to write to cells</param>
|
||||
/// <param name="merge">Whether or not to merge the cells</param>
|
||||
private static void ApplyHeaderFormat(ExcelRange range, string text = null, bool merge = false)
|
||||
{
|
||||
range.Style.HorizontalAlignment = ExcelHorizontalAlignment.Center;
|
||||
range.Style.Font.Bold = true;
|
||||
range.Style.Fill.PatternType = ExcelFillStyle.Solid;
|
||||
range.Style.Fill.BackgroundColor.SetColor(Color.Gainsboro);
|
||||
|
||||
if (!string.IsNullOrEmpty(text))
|
||||
{
|
||||
range.Value = text;
|
||||
}
|
||||
range.Merge = merge;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies write protection security to given worksheet
|
||||
/// </summary>
|
||||
/// <param name="worksheet">Worksheet to apply write protection security to</param>
|
||||
private static void ApplySecurity(ExcelWorksheet worksheet)
|
||||
{
|
||||
worksheet.Protection.IsProtected = true;
|
||||
worksheet.Protection.AllowDeleteColumns = true;
|
||||
worksheet.Protection.AllowDeleteRows = false;
|
||||
worksheet.Protection.AllowAutoFilter = true;
|
||||
worksheet.Protection.AllowFormatCells = true;
|
||||
worksheet.Protection.AllowFormatColumns = true;
|
||||
worksheet.Protection.AllowFormatRows = true;
|
||||
worksheet.Protection.AllowSelectLockedCells = true;
|
||||
worksheet.Protection.AllowSelectUnlockedCells = true;
|
||||
worksheet.Protection.AllowEditObject = true;
|
||||
worksheet.Protection.AllowSort = true;
|
||||
worksheet.Protection.SetPassword("JDESCOPINGTOOL");
|
||||
}
|
||||
}
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace WorkerService.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// Function JSON converter
|
||||
/// </summary>
|
||||
/// <typeparam name="TParameter">Function input type</typeparam>
|
||||
/// <typeparam name="TOutput">Function output type</typeparam>
|
||||
public class FunctionConverter<TParameter, TOutput> : JsonConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes the JSON representation of the object
|
||||
/// </summary>
|
||||
/// <param name="writer">The JsonWriter to write to</param>
|
||||
/// <param name="value">The value to write</param>
|
||||
/// <param name="serializer">The calling serializer</param>
|
||||
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
|
||||
{
|
||||
if (value is Func<TParameter, TOutput> function)
|
||||
{
|
||||
writer.WriteValue($"{function.Method.DeclaringType}.{function.Method.Name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Reads the JSON representation of the object.</summary>
|
||||
/// <param name="reader">JsonReader to read from</param>
|
||||
/// <param name="objectType">Type of the object</param>
|
||||
/// <param name="existingValue">The existing value of object being read</param>
|
||||
/// <param name="serializer">The calling serializer</param>
|
||||
/// <returns>The object value</returns>
|
||||
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
|
||||
{
|
||||
//Get the function's full path
|
||||
string fullFunctionPath = (string)reader.Value;
|
||||
if (string.IsNullOrEmpty(fullFunctionPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
//Extract class and function names
|
||||
string className = fullFunctionPath.Substring(0, fullFunctionPath.LastIndexOf(".", StringComparison.Ordinal));
|
||||
string functionName = fullFunctionPath.Substring(fullFunctionPath.LastIndexOf(".", StringComparison.Ordinal) + 1);
|
||||
|
||||
//Get the class type
|
||||
Type classType = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).FirstOrDefault(t => string.Equals(t.FullName, className, StringComparison.CurrentCultureIgnoreCase));
|
||||
if (classType == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
//Get the function's method info
|
||||
MethodInfo methodInfo = classType.GetMethod(functionName);
|
||||
if (methodInfo == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return (Func<TParameter, TOutput>)methodInfo.CreateDelegate(typeof(Func<TParameter, TOutput>));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether this instance can convert the specified object type
|
||||
/// </summary>
|
||||
/// <param name="objectType">Type of the object</param>
|
||||
/// <returns>
|
||||
/// Whether or not this instance can convert ot the specified object type
|
||||
/// </returns>
|
||||
public override bool CanConvert(Type objectType)
|
||||
{
|
||||
return objectType == typeof(Func<TParameter, TOutput>);
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+191
@@ -0,0 +1,191 @@
|
||||
using System;
|
||||
using System.Configuration;
|
||||
using System.Data;
|
||||
using System.Data.SqlClient;
|
||||
using System.IO;
|
||||
using Dapper;
|
||||
using DataModel.Helpers;
|
||||
using DataModel.Models;
|
||||
using DataModel.Process;
|
||||
using WorkerService.Helpers;
|
||||
using WorkerService.Models.Reporting;
|
||||
|
||||
namespace WorkerService.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// Worker service-specific functionality for LotFinderDB interface
|
||||
/// </summary>
|
||||
public class LotFinderDBExt : LotFinderDB
|
||||
{
|
||||
/// <summary>
|
||||
/// Query to get next queued search
|
||||
/// </summary>
|
||||
private const string SQL_GET_NEXT_SEARCH = @"
|
||||
SELECT TOP 1 s.ID,
|
||||
s.UserName,
|
||||
s.Name,
|
||||
s.Status,
|
||||
s.SubmitDT,
|
||||
s.StartDT,
|
||||
s.EndDT,
|
||||
s.Criteria AS CriteriaJSON
|
||||
FROM dbo.Search s
|
||||
WHERE s.Status = 1
|
||||
ORDER BY s.SubmitDT";
|
||||
|
||||
/// <summary>
|
||||
/// Gets next queued search
|
||||
/// </summary>
|
||||
/// <returns>Next queue search</returns>
|
||||
public static Search GetNextSearch()
|
||||
{
|
||||
Search nextSearch = null;
|
||||
|
||||
try
|
||||
{
|
||||
using (SqlConnection connection = GetConnection())
|
||||
{
|
||||
nextSearch = connection.QueryFirstOrDefault<Search>(SQL_GET_NEXT_SEARCH);
|
||||
|
||||
if (nextSearch != null && !string.IsNullOrEmpty(nextSearch.CriteriaJSON))
|
||||
{
|
||||
nextSearch.Criteria = JsonHelpers.FromJSON<SearchCriteria>(nextSearch.CriteriaJSON);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("GetNextSearch: failed to get next queued search: {0}.", error.Message);
|
||||
}
|
||||
|
||||
return nextSearch;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resets the status of partially completed searches
|
||||
/// </summary>
|
||||
public static void ResetPartialSearches()
|
||||
{
|
||||
try
|
||||
{
|
||||
using (SqlConnection connection = GetConnection())
|
||||
{
|
||||
connection.Execute("ResetPartialSearches", commandType: CommandType.StoredProcedure);
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("ResetPartialSearches: failed to reset partial searches: {0}.", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates the status of the search to 'Start'
|
||||
/// </summary>
|
||||
/// <param name="search">Search to start</param>
|
||||
public static void StartSearch(Search search)
|
||||
{
|
||||
try
|
||||
{
|
||||
search.Status = SearchStatus.Started;
|
||||
search.StartDT = DateTime.Now;
|
||||
|
||||
using (SqlConnection connection = GetConnection())
|
||||
{
|
||||
connection.Execute("StartSearch", new { p_SearchID = search.ID }, commandType: CommandType.StoredProcedure);
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("StartSearch: failed to mark search as started: {0}.", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Update the status of the search to 'Complete' and stores the results
|
||||
/// </summary>
|
||||
/// <param name="search">Search to complete</param>
|
||||
/// <param name="wasSuccessful">Whether or not the search was successful</param>
|
||||
public static void CompleteSearch(Search search, bool wasSuccessful)
|
||||
{
|
||||
try
|
||||
{
|
||||
search.Status = wasSuccessful ? SearchStatus.Ended : SearchStatus.Error;
|
||||
|
||||
using (SqlConnection connection = GetConnection())
|
||||
{
|
||||
connection.Execute("CompleteSearch", new { p_SearchID = search.ID, p_WasSuccessful = wasSuccessful, p_Results = search.Results }, commandType: CommandType.StoredProcedure);
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("CompleteSearch: failed to mark search as completed: {0}.", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs search and extracts results into given model
|
||||
/// </summary>
|
||||
/// <param name="searchModel">Search parameter & result model</param>
|
||||
public static void Search(SearchModel searchModel)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Get configured timeout
|
||||
int queryTimeout = 600;
|
||||
try
|
||||
{
|
||||
string queryTimeoutStr = ConfigurationManager.AppSettings["querytimeout"];
|
||||
queryTimeout = int.Parse(queryTimeoutStr);
|
||||
}
|
||||
catch
|
||||
{
|
||||
//Ignore
|
||||
}
|
||||
|
||||
//Generate query to execute
|
||||
string query = searchModel.GetQuery();
|
||||
File.WriteAllText($"search_{searchModel.ID}.sql", query);
|
||||
|
||||
searchModel.StartDT = DateTime.Now;
|
||||
using (SqlConnection connection = GetConnection())
|
||||
{
|
||||
var results = connection.QueryMultiple(query, new
|
||||
{
|
||||
p_MinimumDT = searchModel.MinimumDT,
|
||||
p_MaximumDT = searchModel.MaximumDT,
|
||||
p_WorkOrderFilter = searchModel.CreateWorkOrderFilterParameter(),
|
||||
p_ItemNumberFilter = searchModel.CreateItemNumberFilterParameter(),
|
||||
p_ProfitCenterFilter = searchModel.CreateProfitCenterFilterParameter(),
|
||||
p_WorkCenterFilter = searchModel.CreateWorkCenterFilterParameter(),
|
||||
p_ComponentLotFilter = searchModel.CreateComponentLotFilterParameter(),
|
||||
p_OperatorFilter = searchModel.CreateOperatorFilterParameter(),
|
||||
p_ItemOperationMisFilter = searchModel.CreateItemOperationMisFilterParameter(),
|
||||
p_ExtractMisData = searchModel.ExtractMisData
|
||||
}, commandTimeout: queryTimeout);
|
||||
|
||||
//Parse search results
|
||||
searchModel.Results.AddRange(results.Read<SearchResult>());
|
||||
|
||||
//Parse MIS data if extracted
|
||||
if (searchModel.ExtractMisData)
|
||||
{
|
||||
searchModel.MisResults.AddRange(results.Read<MisSearchResult>());
|
||||
searchModel.MisNonMatchResults.AddRange(results.Read<MisNonMatchSearchResult>());
|
||||
}
|
||||
}
|
||||
searchModel.EndDT = DateTime.Now;
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("Search: failed to perform search: {0}.", error.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data.SqlClient;
|
||||
using System.Linq;
|
||||
using Dapper;
|
||||
using DataModel.Models;
|
||||
using DataModel.Process;
|
||||
using WorkerService.Models;
|
||||
|
||||
namespace WorkerService.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// Data update entry management functionality for data update processor
|
||||
/// </summary>
|
||||
public partial class UpdateProcessor
|
||||
{
|
||||
/// <summary>
|
||||
/// SQL to close any open update entries (identified by number records = -2)
|
||||
/// </summary>
|
||||
private const string SQL_CLOSE_OPEN_UPDATE_ENTRIES = @"
|
||||
UPDATE dbo.DataUpdate
|
||||
SET EndDT = GETDATE(), WasSuccessful = 0, NumberRecords = -1
|
||||
WHERE NumberRecords = -2";
|
||||
|
||||
/// <summary>
|
||||
/// Closes any open data update entries
|
||||
/// </summary>
|
||||
private static void CloseOpenUpdateEntries()
|
||||
{
|
||||
try
|
||||
{
|
||||
using (SqlConnection connection = LotFinderDB.GetConnection())
|
||||
{
|
||||
//Close any open update entries
|
||||
connection.Execute(SQL_CLOSE_OPEN_UPDATE_ENTRIES);
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("CloseOpenUpdateEntries: failed to close open data update entries: {0}.", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SQL to purge data update records
|
||||
/// </summary>
|
||||
private const string SQL_PURGE_UPDATE_ENTRIES = @"
|
||||
DELETE FROM dbo.DataUpdate
|
||||
WHERE StartDT < DATEADD(DAY, @maxAge * -1, GETDATE())";
|
||||
|
||||
/// <summary>
|
||||
/// Purges any data update entries older than given max age
|
||||
/// </summary>
|
||||
/// <param name="maxAge">Maximum entry age (in days)</param>
|
||||
private static void PurgeUpdateEntries(int maxAge)
|
||||
{
|
||||
try
|
||||
{
|
||||
using (SqlConnection connection = LotFinderDB.GetConnection())
|
||||
{
|
||||
//Purge the records
|
||||
connection.Execute(SQL_PURGE_UPDATE_ENTRIES, new { maxAge });
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("PurgeUpdateEntries: failed to purge data update entries older than {0} days: {1}.", maxAge, error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SQL to insert data update record
|
||||
/// </summary>
|
||||
private const string SQL_LOG_DATA_UPDATE_START = @"
|
||||
INSERT INTO dbo.DataUpdate(SourceSystem, SourceData, TableName, StartDT, EndDT, UpdateType, WasSuccessful, NumberRecords)
|
||||
OUTPUT INSERTED.*
|
||||
VALUES(@sourceSystem, @sourceData, @tableName, GETDATE(), '1970-01-01', @updateType, 0, -2);";
|
||||
|
||||
/// <summary>
|
||||
/// Logs the data update entry at the start of the update
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="dataUpdate">Data update entry to log</param>
|
||||
public static void LogDataUpdateStart(SqlConnection connection, DataUpdate dataUpdate)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Update the record
|
||||
DataUpdate inserted = connection.QueryFirst<DataUpdate>(SQL_LOG_DATA_UPDATE_START,
|
||||
new
|
||||
{
|
||||
sourceSystem = dataUpdate.SourceSystem,
|
||||
sourceData = dataUpdate.SourceData,
|
||||
tableName = dataUpdate.TableName,
|
||||
updateType = dataUpdate.UpdateType
|
||||
});
|
||||
|
||||
//Copy output values to model
|
||||
dataUpdate.ID = inserted.ID;
|
||||
dataUpdate.StartDT = inserted.StartDT;
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("LogDataUpdateStart: failed to log starting data update entry: {0}.", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SQL to update data update record with results
|
||||
/// </summary>
|
||||
private const string SQL_LOG_DATA_UPDATE_END = @"
|
||||
UPDATE dbo.DataUpdate
|
||||
SET EndDT = GETDATE(), WasSuccessful = @wasSuccessful, NumberRecords = @numberRecords
|
||||
OUTPUT INSERTED.*
|
||||
WHERE ID = @id";
|
||||
|
||||
/// <summary>
|
||||
/// Logs the data update entry at the end of the update
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="dataUpdate">Data update entry to log</param>
|
||||
public static void LogDataUpdateEnd(SqlConnection connection, DataUpdate dataUpdate)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Update the record
|
||||
DataUpdate updated = connection.QueryFirst<DataUpdate>(SQL_LOG_DATA_UPDATE_END,
|
||||
new
|
||||
{
|
||||
id = dataUpdate.ID,
|
||||
wasSuccessful = dataUpdate.WasSuccessful,
|
||||
numberRecords = dataUpdate.NumberRecords
|
||||
});
|
||||
|
||||
//Copy output values to model
|
||||
dataUpdate.EndDT = updated.EndDT;
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("LogDataUpdateEnd: failed to log ending data update entry: {0}.", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets set of last successful data updates for all tables
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <returns>Set of last successful data updates for all tables</returns>
|
||||
private static List<LastDataUpdate> GetLastDataUpdates(SqlConnection connection)
|
||||
{
|
||||
return connection.Query<LastDataUpdate>("SELECT * FROM dbo.LastDataUpdates").ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data.SqlClient;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using Dapper;
|
||||
using WorkerService.Models;
|
||||
|
||||
namespace WorkerService.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// Table management functionality for data update processor
|
||||
/// </summary>
|
||||
public partial class UpdateProcessor
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates staging table with matching column layout of given table
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="tableName">Name of table to create staging table for</param>
|
||||
/// <returns>Name of created temporary table</returns>
|
||||
public static string CreateStagingTable(SqlConnection connection, string tableName)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Get table specification
|
||||
TableSpec tableSpec = GetTableSpec(connection, tableName);
|
||||
|
||||
//Drop temp table if it already exists
|
||||
connection.Execute($"IF OBJECT_ID('tempdb..{tableSpec.StagingTableName}') IS NOT NULL DROP TABLE {tableSpec.StagingTableName};");
|
||||
|
||||
//Create temp table
|
||||
connection.Execute($"CREATE TABLE {tableSpec.StagingTableName}({string.Join(",", tableSpec.Columns.Select(c => $"{c.Name} {c.Definition}"))});");
|
||||
|
||||
//Create indicies on temp table
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.Append($"CREATE INDEX IDX_STAGING_{tableSpec.Name} ON {tableSpec.StagingTableName}(");
|
||||
builder.Append(string.Join(",", tableSpec.PrimaryKey.Select(c => $"{c.Name}")));
|
||||
if (tableSpec.Columns.Any(c => c.Name.Equals("LastUpdateDT", StringComparison.CurrentCultureIgnoreCase)))
|
||||
{
|
||||
builder.Append(", LastUpdateDT DESC");
|
||||
}
|
||||
else if (tableSpec.Columns.Any(c => c.Name.Equals("ReleaseDate", StringComparison.CurrentCultureIgnoreCase)))
|
||||
{
|
||||
builder.Append(", ReleaseDate DESC");
|
||||
}
|
||||
builder.Append(");");
|
||||
|
||||
connection.Execute(builder.ToString());
|
||||
|
||||
//Disable indicies on temp table
|
||||
DisableIndicies(connection, tableSpec.StagingTableName);
|
||||
|
||||
return tableSpec.StagingTableName;
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log and forward error
|
||||
logger.Error("GetStagingTable: failed to create staging table for '{0}': {1}.", tableName, error.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates temporary table with matching column layout of given table
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="tableName">Name of table to create temporary table for</param>
|
||||
/// <returns>Name of created temporary table</returns>
|
||||
public static string CreateTempTable(SqlConnection connection, string tableName)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Get table specification
|
||||
TableSpec tableSpec = GetTableSpec(connection, tableName);
|
||||
|
||||
//Drop temp table if it already exists
|
||||
connection.Execute($"IF OBJECT_ID('tempdb..{tableSpec.TempTableName}') IS NOT NULL DROP TABLE {tableSpec.TempTableName};");
|
||||
|
||||
//Create temp table
|
||||
connection.Execute($"CREATE TABLE {tableSpec.TempTableName}({string.Join(",", tableSpec.Columns.Select(c => $"{c.Name} {c.Definition}"))}, CONSTRAINT PK_{tableSpec.TempTableName} PRIMARY KEY CLUSTERED({string.Join(",", tableSpec.PrimaryKey.Select(c => $"{c.Name}"))}));");
|
||||
|
||||
StringBuilder builder = new StringBuilder();
|
||||
|
||||
builder.AppendLine("WITH StagingCTE AS (");
|
||||
builder.Append($"SELECT st.*, ROW_NUMBER() OVER(PARTITION BY {string.Join(", ", tableSpec.PrimaryKey.Select(c=>c.Name))} ORDER BY {tableSpec.Columns.FirstOrDefault(c => c.Name.Equals("LastUpdateDT", StringComparison.CurrentCultureIgnoreCase) || c.Name.Equals("ReleaseDate", StringComparison.CurrentCultureIgnoreCase))?.Name}) RN FROM {tableSpec.StagingTableName} st");
|
||||
builder.AppendLine(")");
|
||||
builder.AppendLine($"INSERT INTO {tableSpec.TempTableName}({string.Join(", ", tableSpec.Columns.Select(c=>c.Name))})");
|
||||
builder.AppendLine($"SELECT {string.Join(", ", tableSpec.Columns.Select(c => c.Name))} FROM StagingCTE WHERE RN = 1 ORDER BY {string.Join(", ", tableSpec.PrimaryKey.Select(c=>c.Name))};");
|
||||
|
||||
connection.Execute(builder.ToString(),commandTimeout:600);
|
||||
|
||||
return tableSpec.TempTableName;
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log and forward error
|
||||
logger.Error("CreateTableTable: failed to create temporary table for '{0}': {1}.", tableName, error.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates merge statement for given table
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="tableName">Name of table to generate merge statement for</param>
|
||||
/// <returns>Merge statement for given table</returns>
|
||||
private static string GenerateMerge(SqlConnection connection, string tableName)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Get table specification
|
||||
TableSpec tableSpec = GetTableSpec(connection, tableName);
|
||||
|
||||
StringBuilder builder = new StringBuilder();
|
||||
|
||||
builder.AppendFormat("MERGE {0} AS TARGET", tableSpec.Name);
|
||||
builder.AppendLine("");
|
||||
|
||||
builder.AppendFormat("USING {0} AS SOURCE ON({1})", tableSpec.TempTableName, string.Join(" AND ", tableSpec.PrimaryKey.Select(c => $"TARGET.{c.Name} = SOURCE.{c.Name}")));
|
||||
builder.AppendLine("");
|
||||
|
||||
builder.Append("WHEN MATCHED");
|
||||
if (tableSpec.Columns.Exists(c => c.Name.Equals("LastUpdateDT", StringComparison.CurrentCultureIgnoreCase)))
|
||||
{
|
||||
builder.Append(" AND TARGET.LastUpdateDT < SOURCE.LastUpdateDT");
|
||||
}
|
||||
builder.AppendLine(" THEN");
|
||||
|
||||
builder.Append("UPDATE SET ");
|
||||
builder.Append(string.Join(", ", tableSpec.Columns.Where(c => !tableSpec.PrimaryKey.Contains(c)).Select(c => $"TARGET.{c.Name} = SOURCE.{c.Name}")));
|
||||
builder.AppendLine();
|
||||
|
||||
builder.AppendLine("WHEN NOT MATCHED BY TARGET THEN");
|
||||
|
||||
builder.Append("INSERT(");
|
||||
builder.Append(string.Join(", ", tableSpec.Columns.Select(c => c.Name)));
|
||||
builder.AppendLine(")");
|
||||
|
||||
builder.Append("VALUES(");
|
||||
builder.Append(string.Join(", ", tableSpec.Columns.Select(c => $"SOURCE.{c.Name}")));
|
||||
builder.AppendLine(");");
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log and forward error
|
||||
logger.Error("GenerateMerge: failed to generate merge statement for '{0}': {1}.", tableName, error.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates bulk copy specification for given table
|
||||
/// </summary>
|
||||
/// <typeparam name="T">Data type being bulk copied</typeparam>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="tableName">Name of table to generate bulk copy for</param>
|
||||
/// <returns>Bulk copy for given table</returns>
|
||||
private static SqlBulkCopy GenerateBulkCopy<T>(SqlConnection connection, string tableName)
|
||||
{
|
||||
return GenerateBulkCopy(connection, tableName, typeof(T));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates bulk copy specification for given table
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="tableName">Name of table to generate bulk copy for</param>
|
||||
/// <param name="type">Data type being bulk copied</param>
|
||||
/// <returns>Bulk copy for given table</returns>
|
||||
private static SqlBulkCopy GenerateBulkCopy(SqlConnection connection, string tableName, Type type)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Get table specification
|
||||
TableSpec tableSpec = GetTableSpec(connection, tableName);
|
||||
|
||||
//Get class properties
|
||||
List<string> properties = type.GetProperties(BindingFlags.Instance | BindingFlags.Public).Select(p => p.Name).ToList();
|
||||
|
||||
//Build bulk copy specification
|
||||
SqlBulkCopy bulkCopy = new SqlBulkCopy(connection)
|
||||
{
|
||||
BatchSize = 10000,
|
||||
NotifyAfter = 5000,
|
||||
EnableStreaming = true,
|
||||
DestinationTableName = tableSpec.StagingTableName
|
||||
};
|
||||
|
||||
foreach (ColumnSpec columnSpec in tableSpec.Columns)
|
||||
{
|
||||
string property = properties.FirstOrDefault(p => p.Equals(columnSpec.Name, StringComparison.CurrentCultureIgnoreCase));
|
||||
if (!string.IsNullOrEmpty(property))
|
||||
{
|
||||
bulkCopy.ColumnMappings.Add(property, columnSpec.Name);
|
||||
}
|
||||
}
|
||||
|
||||
return bulkCopy;
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log and forward error
|
||||
logger.Error("GenerateBulkCopy: failed to generate bulk copy for '{0}': {1}.", tableName, error.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SQL to get the columns for the given table
|
||||
/// </summary>
|
||||
private const string SQL_GET_TABLE_COLUMNS = @"
|
||||
SELECT c.name AS Name,
|
||||
CASE t2.name
|
||||
WHEN 'char' THEN 'CHAR(' + CAST(c.max_length AS VARCHAR(10)) + ')'
|
||||
WHEN 'varchar' THEN 'VARCHAR(' + CASE c.max_length WHEN -1 THEN 'MAX' ELSE CAST(c.max_length AS VARCHAR(10)) END + ')'
|
||||
WHEN 'decimal' THEN 'DECIMAL(' + CAST(c.precision AS VARCHAR(4)) + ',' + CAST(c.scale AS VARCHAR(4)) + ')'
|
||||
ELSE UPPER(t2.name)
|
||||
END AS Definition
|
||||
FROM sys.columns c INNER JOIN
|
||||
sys.types AS t2 ON (c.system_type_id = t2.system_type_id) INNER JOIN
|
||||
sys.tables t ON (c.object_id = t.object_id)
|
||||
WHERE t.name = @tableName
|
||||
ORDER BY c.column_id";
|
||||
|
||||
/// <summary>
|
||||
/// SQL to get the primary key columns for the given table
|
||||
/// </summary>
|
||||
private const string SQL_GET_TABLE_PRIMARY_KEY = @"
|
||||
SELECT COLUMN_NAME AS Name
|
||||
FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
|
||||
WHERE OBJECTPROPERTY(OBJECT_ID(CONSTRAINT_SCHEMA + '.' + QUOTENAME(CONSTRAINT_NAME)), 'IsPrimaryKey') = 1 AND
|
||||
TABLE_NAME = @tableName
|
||||
ORDER BY ORDINAL_POSITION";
|
||||
|
||||
/// <summary>
|
||||
/// Gets the table specification for the given table
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="tableName">Name of table to get specification for</param>
|
||||
/// <returns>Table specification for the given table</returns>
|
||||
public static TableSpec GetTableSpec(SqlConnection connection, string tableName)
|
||||
{
|
||||
TableSpec tableSpec = new TableSpec() { Name = tableName };
|
||||
|
||||
//Load columns
|
||||
tableSpec.Columns.AddRange(connection.Query<ColumnSpec>(SQL_GET_TABLE_COLUMNS, new { tableName }));
|
||||
|
||||
//Load primary key
|
||||
tableSpec.PrimaryKey.AddRange(connection.Query<string>(SQL_GET_TABLE_PRIMARY_KEY, new { tableName }).Select(cn => tableSpec.Columns.First(c => c.Name.Equals(cn, StringComparison.CurrentCultureIgnoreCase))));
|
||||
|
||||
return tableSpec;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Truncates given table
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="tableName">Name of table to truncate</param>
|
||||
private static void TruncateTable(SqlConnection connection, string tableName)
|
||||
{
|
||||
try
|
||||
{
|
||||
logger.Debug("TruncateTable: truncating table '{0}'", tableName);
|
||||
|
||||
//Generate and execute SQL to truncate table
|
||||
string sql = $"TRUNCATE TABLE {tableName};";
|
||||
connection.Execute(sql);
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("TruncateTable: failed to truncate table '{0}': {1}.", tableName, error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SQL to get indices on given table
|
||||
/// </summary>
|
||||
private const string SQL_GET_INDICES = @"
|
||||
SELECT DISTINCT
|
||||
ind.name AS Name,
|
||||
ind.is_primary_key AS IsPrimaryKey,
|
||||
ind.is_unique AS IsUnique,
|
||||
ind.is_unique_constraint AS IsUniqueConstraint
|
||||
FROM sys.indexes ind INNER JOIN
|
||||
sys.index_columns ic ON (ind.object_id = ic.object_id AND ind.index_id = ic.index_id) INNER JOIN
|
||||
sys.columns col ON (ic.object_id = col.object_id AND ic.column_id = col.column_id) INNER JOIN
|
||||
sys.tables t ON (ind.object_id = t.object_id)
|
||||
WHERE t.name = @tableName";
|
||||
|
||||
/// <summary>
|
||||
/// Disables all non-PK indices on given table
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>
|
||||
/// <param name="tableName">Name of table to disable non-PK indices for</param>
|
||||
private static void DisableIndicies(SqlConnection connection, string tableName)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Get all indices on table
|
||||
List<Index> indices = connection.Query<Index>(SQL_GET_INDICES, new { tableName }).ToList();
|
||||
|
||||
//Loop through all non-PK/non-cluster indices
|
||||
foreach (Index index in indices.Where(i => !i.IsPrimaryKey && !i.IsUnique && !i.IsUniqueConstraint))
|
||||
{
|
||||
//Generate and execute SQL to disable index
|
||||
string sql = $"ALTER INDEX {index.Name} ON {tableName} DISABLE;";
|
||||
connection.Execute(sql);
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("DisableIndicies: failed to disable non-PK indicies on table '{0}': {1}.", tableName, error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rebuilds all indices on given table
|
||||
/// </summary>
|
||||
/// <param name="connection">SQL connection to execute commands on</param>So
|
||||
/// <param name="tableName">Name of table to rebuild indices for</param>
|
||||
private static void RebuildIndicies(SqlConnection connection, string tableName)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Get all indices on table
|
||||
List<Index> indices = connection.Query<Index>(SQL_GET_INDICES, new { tableName }).ToList();
|
||||
|
||||
//Loop through indices
|
||||
foreach (Index index in indices.Where(i => !i.IsPrimaryKey && !i.IsUnique && !i.IsUniqueConstraint))
|
||||
{
|
||||
//Generate and execute SQL to rebuild index
|
||||
string sql = $"ALTER INDEX {index.Name} ON {tableName} REBUILD;";
|
||||
connection.Execute(sql);
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("RebuildIndicies: failed to rebuild indicies on table '{0}': {1}.", tableName, error.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+220
@@ -0,0 +1,220 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Data.SqlClient;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Dapper;
|
||||
using DataModel.Helpers;
|
||||
using DataModel.Models;
|
||||
using DataModel.Process;
|
||||
using Newtonsoft.Json;
|
||||
using NLog;
|
||||
using WorkerService.Models;
|
||||
|
||||
namespace WorkerService.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// Data update processor
|
||||
/// </summary>
|
||||
public partial class UpdateProcessor
|
||||
{
|
||||
/// <summary>
|
||||
/// Shared logger instance
|
||||
/// </summary>
|
||||
private static readonly Logger logger = LogManager.GetCurrentClassLogger();
|
||||
|
||||
/// <summary>
|
||||
/// Collection of configured data source update configurations
|
||||
/// </summary>
|
||||
public static readonly List<DataSourceConfig> configs = new List<DataSourceConfig>();
|
||||
|
||||
/// <summary>
|
||||
/// Static class initializer
|
||||
/// </summary>
|
||||
static UpdateProcessor()
|
||||
{
|
||||
foreach (string configFileName in Directory.GetFiles("dsconfig"))
|
||||
{
|
||||
//continue;
|
||||
DataSourceConfig config = JsonConvert.DeserializeObject<DataSourceConfig>(File.ReadAllText(configFileName));
|
||||
|
||||
if (config.IsEnabled) { configs.Add(config); }
|
||||
}
|
||||
}
|
||||
|
||||
public static void DoUpdate(string tableName, UpdateTypes updateType = UpdateTypes.Mass)
|
||||
{
|
||||
DataSourceConfig dataSourceConfig = configs.FirstOrDefault(c => c.TableName.Equals(tableName, StringComparison.CurrentCultureIgnoreCase));
|
||||
|
||||
//Get last data updates
|
||||
List<LastDataUpdate> lastDataUpdates = null;
|
||||
using (SqlConnection connection = LotFinderDB.GetConnection())
|
||||
{
|
||||
lastDataUpdates = GetLastDataUpdates(connection);
|
||||
}
|
||||
|
||||
LastDataUpdate lastDataUpdate = lastDataUpdates.FirstOrDefault(ldu => ldu.TableName.Equals(dataSourceConfig.TableName));
|
||||
DateTime? minDT = null;
|
||||
switch (updateType)
|
||||
{
|
||||
case UpdateTypes.Mass:
|
||||
minDT = null;
|
||||
break;
|
||||
case UpdateTypes.Daily:
|
||||
minDT = lastDataUpdate.DailyUpdateDT.AddMinutes(-3 * dataSourceConfig.DailyUpdateConfig.Interval);
|
||||
break;
|
||||
case UpdateTypes.Hourly:
|
||||
minDT = lastDataUpdate.HourlyUpdateDT.AddMinutes(-3 * dataSourceConfig.HourlyUpdateConfig.Interval);
|
||||
break;
|
||||
}
|
||||
|
||||
DoUpdate(dataSourceConfig, updateType, minDT);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets list of pending data update tasks
|
||||
/// </summary>
|
||||
/// <returns>Pending data update tasks</returns>
|
||||
public static List<DataUpdateTask> GetPendingUpdateTasks()
|
||||
{
|
||||
List<DataUpdateTask> pending = new List<DataUpdateTask>();
|
||||
|
||||
using (SqlConnection connection = LotFinderDB.GetConnection())
|
||||
{
|
||||
//Get last data updates
|
||||
List<LastDataUpdate> lastDataUpdates = GetLastDataUpdates(connection);
|
||||
|
||||
foreach (DataSourceConfig dataSourceConfig in configs)
|
||||
{
|
||||
LastDataUpdate lastDataUpdate = lastDataUpdates.FirstOrDefault(ldu => ldu.TableName.Equals(dataSourceConfig.TableName));
|
||||
|
||||
if (lastDataUpdate == null || (dataSourceConfig.MassUpdateConfig.Enabled && DateTime.Now > lastDataUpdate.MassUpdateDT.AddMinutes(dataSourceConfig.MassUpdateConfig.Interval)))
|
||||
{
|
||||
pending.Add(new DataUpdateTask() { Configuration = dataSourceConfig, UpdateType = UpdateTypes.Mass });
|
||||
}
|
||||
else if (dataSourceConfig.DailyUpdateConfig.Enabled && DateTime.Now > lastDataUpdate.DailyUpdateDT.AddMinutes(dataSourceConfig.DailyUpdateConfig.Interval))
|
||||
{
|
||||
pending.Add(new DataUpdateTask() { Configuration = dataSourceConfig, UpdateType = UpdateTypes.Daily, MinimumDT = lastDataUpdate.DailyUpdateDT.AddMinutes(-3 * dataSourceConfig.DailyUpdateConfig.Interval) });
|
||||
}
|
||||
else if (dataSourceConfig.HourlyUpdateConfig.Enabled && DateTime.Now > lastDataUpdate.HourlyUpdateDT.AddMinutes(dataSourceConfig.HourlyUpdateConfig.Interval))
|
||||
{
|
||||
pending.Add(new DataUpdateTask() { Configuration = dataSourceConfig, UpdateType = UpdateTypes.Hourly, MinimumDT = lastDataUpdate.DailyUpdateDT.AddMinutes(-3 * dataSourceConfig.DailyUpdateConfig.Interval) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pending;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs the data update
|
||||
/// </summary>
|
||||
/// <param name="dataUpdateTask">Data update task to execute</param>
|
||||
/// <returns>Data update results</returns>
|
||||
public static DataUpdate DoUpdate(DataUpdateTask dataUpdateTask)
|
||||
{
|
||||
logger.Info($"Starting [{dataUpdateTask.UpdateType}] data update for {dataUpdateTask.Configuration.TableName}...");
|
||||
return DoUpdate(dataUpdateTask.Configuration, dataUpdateTask.UpdateType, dataUpdateTask.MinimumDT);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs the data update
|
||||
/// </summary>
|
||||
/// <param name="config">Data source configuration</param>
|
||||
/// <param name="updateType">Type of update to perform</param>
|
||||
/// <param name="minDT">Minimum timestamp to update data from</param>
|
||||
/// <returns>Data update results</returns>
|
||||
public static DataUpdate DoUpdate(DataSourceConfig config, UpdateTypes updateType, DateTime? minDT)
|
||||
{
|
||||
//Log start of data update
|
||||
DataUpdate dataUpdate = new DataUpdate()
|
||||
{
|
||||
SourceSystem = config.SourceSystem,
|
||||
SourceData = config.SourceData,
|
||||
TableName = config.TableName,
|
||||
UpdateType = updateType,
|
||||
StartDT = DateTime.Now,
|
||||
NumberRecords = 0
|
||||
};
|
||||
|
||||
//Get data update configuration details
|
||||
DataUpdateConfig updateConfig;
|
||||
switch (updateType)
|
||||
{
|
||||
case UpdateTypes.Hourly:
|
||||
updateConfig = config.HourlyUpdateConfig;
|
||||
break;
|
||||
case UpdateTypes.Daily:
|
||||
updateConfig = config.DailyUpdateConfig;
|
||||
break;
|
||||
case UpdateTypes.Mass:
|
||||
updateConfig = config.MassUpdateConfig;
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(updateType), updateType, null);
|
||||
}
|
||||
|
||||
using (SqlConnection connection = LotFinderDB.GetConnection())
|
||||
{
|
||||
LogDataUpdateStart(connection, dataUpdate);
|
||||
|
||||
//Clear destination table if needed
|
||||
if (updateConfig.PrepurgeData)
|
||||
{
|
||||
TruncateTable(connection, config.TableName);
|
||||
}
|
||||
|
||||
Type sourceType = config.DataFetchFunction.Method.ReturnType.GenericTypeArguments[0];
|
||||
|
||||
//Fetch data
|
||||
IEnumerable<dynamic> data = config.DataFetchFunction(minDT);
|
||||
|
||||
//Generate SQL to merge temp data to destination table
|
||||
string mergeSQL = GenerateMerge(connection, config.TableName);
|
||||
|
||||
foreach (var batch in data.BatchGroup(1000000))
|
||||
{
|
||||
//Setup temp table
|
||||
string stagingTableName = CreateStagingTable(connection, config.TableName);
|
||||
|
||||
//Copy data to temp table
|
||||
SqlBulkCopy bulkCopy = GenerateBulkCopy(connection, config.TableName, sourceType);
|
||||
IDataReader reader = new GenericListDataReader(batch, sourceType);
|
||||
bulkCopy.WriteToServer(reader);
|
||||
dataUpdate.NumberRecords += batch.Count;
|
||||
|
||||
//Index temp table
|
||||
RebuildIndicies(connection, stagingTableName);
|
||||
|
||||
//Copy to temp table
|
||||
string tempTableName = CreateTempTable(connection, config.TableName);
|
||||
|
||||
//Merge data from temp table to destination table
|
||||
connection.Execute(mergeSQL, commandTimeout: 6000);
|
||||
|
||||
logger.Debug("DoUpdate: {0:n0} rows merged to {1}", dataUpdate.NumberRecords, config.TableName);
|
||||
}
|
||||
|
||||
//Run post processing action if configured
|
||||
if (config.PostProcessingAction != null)
|
||||
{
|
||||
config.PostProcessingAction();
|
||||
}
|
||||
|
||||
//Re-index destination table if needed
|
||||
if (updateConfig.ReIndexData)
|
||||
{
|
||||
RebuildIndicies(connection, dataUpdate.TableName);
|
||||
}
|
||||
|
||||
//Update data update entry
|
||||
dataUpdate.WasSuccessful = true;
|
||||
LogDataUpdateEnd(connection, dataUpdate);
|
||||
}
|
||||
|
||||
return dataUpdate;
|
||||
}
|
||||
}
|
||||
}
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data.SQLite;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Commons.Helpers;
|
||||
using Commons.Models;
|
||||
using Commons.Process;
|
||||
using Dapper;
|
||||
|
||||
namespace WorkerService.Process
|
||||
{
|
||||
public partial class Updater
|
||||
{
|
||||
private const string SQL_GET_MIS_DATA = @"
|
||||
select
|
||||
ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID,
|
||||
CharNumber, TestGroup, TestDefinition, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions,
|
||||
Status, ReleaseDate, ObsoleteDate
|
||||
FROM infodb.MisInfo info INNER JOIN
|
||||
relationdb.MisRelation rel ON (info.RevFK = rel.RevFK) INNER JOIN
|
||||
detaildb.MisDetail det ON (rel.MisFK = det.MisFK) INNER JOIN
|
||||
partdb.MisPart part ON (det.PartFK = part.PartFK)
|
||||
ORDER BY ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, Status, CharNumber";
|
||||
|
||||
public static IEnumerable<MisData> GetMisData()
|
||||
{
|
||||
Task misInfoTask = Task.Run(() => PrepareMisInfo());
|
||||
Task misRelationTask = Task.Run(() => PrepareMisRelation());
|
||||
Task misDetailTask = Task.Run(() => PrepareMisDetail());
|
||||
Task misPartTask = Task.Run(() => PrepareMisPart());
|
||||
|
||||
Task.WaitAll(misInfoTask, misRelationTask, misDetailTask, misPartTask);
|
||||
|
||||
if (File.Exists("misdata.db"))
|
||||
{
|
||||
File.Delete("misdata.db");
|
||||
}
|
||||
SQLiteConnection.CreateFile("misdata.db");
|
||||
using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misdata.db;Version=3;Journal Mode=Off;"))
|
||||
{
|
||||
sqlite.Open();
|
||||
|
||||
//Attach databases
|
||||
sqlite.Execute("ATTACH DATABASE 'misinfo.db' AS infodb;");
|
||||
sqlite.Execute("ATTACH DATABASE 'misrelation.db' AS relationdb;");
|
||||
sqlite.Execute("ATTACH DATABASE 'misdetail.db' AS detaildb;");
|
||||
sqlite.Execute("ATTACH DATABASE 'mispart.db' AS partdb;");
|
||||
|
||||
foreach (MisData misData in sqlite.Query<MisData>(SQL_GET_MIS_DATA, buffered: false))
|
||||
{
|
||||
yield return misData;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void PrepareMisInfo()
|
||||
{
|
||||
return;
|
||||
if (File.Exists("misinfo.db"))
|
||||
{
|
||||
File.Delete("misinfo.db");
|
||||
}
|
||||
SQLiteConnection.CreateFile("misinfo.db");
|
||||
|
||||
//Fetch MIS info
|
||||
List<MisInfo> misInfos = CMS.GetMisInfos().ToList();
|
||||
|
||||
//Lookup obsolete date by backlevel release
|
||||
foreach (var group in misInfos.GroupBy(m => new { m.MisNumber, m.RevID }))
|
||||
{
|
||||
DateTime? obsoleteDate = group.Where(g => g.Status.Equals("BackLevel") && g.ReleaseDate != null).Select(g => g.ReleaseDate).FirstOrDefault();
|
||||
|
||||
foreach (MisInfo current in group.Where(g => g.Status.Equals("Current")))
|
||||
{
|
||||
current.ObsoleteDate = obsoleteDate;
|
||||
}
|
||||
}
|
||||
|
||||
//Lookup obsolete date by next revision release
|
||||
var lookup = misInfos.Where(m => m.ReleaseDate != null).GroupBy(m => new { m.MisNumber, m.RevID, m.Status }).Select(g => new { g.Key.MisNumber, g.Key.RevID, g.Key.Status, ReleaseDate = g.Min(m => m.ReleaseDate) }).ToDictionary(g => new { g.MisNumber, g.RevID, g.Status }, g => g.ReleaseDate);
|
||||
foreach (MisInfo misInfo in misInfos.Where(m => !m.ObsoleteDate.HasValue))
|
||||
{
|
||||
DateTime? obsoleteDate;
|
||||
if (lookup.TryGetValue(new { misInfo.MisNumber, misInfo.RevID, misInfo.Status }, out obsoleteDate))
|
||||
{
|
||||
misInfo.ObsoleteDate = obsoleteDate;
|
||||
}
|
||||
}
|
||||
|
||||
//Write MIS info to database
|
||||
using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misinfo.db;Version=3;Journal Mode=Off;"))
|
||||
{
|
||||
sqlite.Open();
|
||||
|
||||
//Create table for MIS info
|
||||
sqlite.Execute("CREATE TABLE MisInfo(RevFK TEXT, MisNumber TEXT, RevID Text, Status TEXT, ReleaseDate DATETIME, ObsoleteDate DATETIME);");
|
||||
|
||||
foreach (var batch in misInfos.BatchGroup(10000))
|
||||
{
|
||||
using (SQLiteTransaction transaction = sqlite.BeginTransaction())
|
||||
{
|
||||
foreach (MisInfo misInfo in batch)
|
||||
{
|
||||
sqlite.Execute("INSERT INTO MisInfo(RevFK, MisNumber, RevID, Status, ReleaseDate, ObsoleteDate) VALUES(@RevFK, @MisNumber, @RevID, @Status, @ReleaseDate, @ObsoleteDate)", misInfo);
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
}
|
||||
|
||||
//Add index on FK field
|
||||
sqlite.Execute("CREATE INDEX IDX_MisInfo_RevFK ON MisInfo(RevFK);");
|
||||
|
||||
sqlite.Execute("VACUUM");
|
||||
}
|
||||
}
|
||||
|
||||
private static void PrepareMisRelation()
|
||||
{
|
||||
return;
|
||||
if (File.Exists("misrelation.db"))
|
||||
{
|
||||
File.Delete("misrelation.db");
|
||||
}
|
||||
SQLiteConnection.CreateFile("misrelation.db");
|
||||
|
||||
IEnumerable<MisRelation> misRelations = CMS.GetMisRelations();
|
||||
|
||||
//Write MIS relation to database
|
||||
using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misrelation.db;Version=3;Journal Mode=Off;"))
|
||||
{
|
||||
sqlite.Open();
|
||||
|
||||
//Create table for MIS relation
|
||||
sqlite.Execute("CREATE TABLE MisRelation(RevFK TEXT, MisFK TEXT);");
|
||||
|
||||
foreach (var batch in misRelations.BatchGroup(10000))
|
||||
{
|
||||
using (SQLiteTransaction transaction = sqlite.BeginTransaction())
|
||||
{
|
||||
foreach (MisRelation misRelation in batch)
|
||||
{
|
||||
sqlite.Execute("INSERT INTO MisRelation(RevFK, MisFK) VALUES(@RevFK, @MisFK)", misRelation);
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
}
|
||||
|
||||
//Add indices on FK fields
|
||||
sqlite.Execute("CREATE INDEX IDX_MisRelation_RevFK ON MisRelation(RevFK);");
|
||||
sqlite.Execute("CREATE INDEX IDX_MisRelation_MisFK ON MisRelation(MisFK);");
|
||||
|
||||
sqlite.Execute("VACUUM");
|
||||
}
|
||||
}
|
||||
|
||||
public static void PrepareMisDetail()
|
||||
{
|
||||
return;
|
||||
if (File.Exists("misdetail.db"))
|
||||
{
|
||||
File.Delete("misdetail.db");
|
||||
}
|
||||
SQLiteConnection.CreateFile("misdetail.db");
|
||||
|
||||
IEnumerable<MisDetail> misDetails = CMS.GetMisDetails();
|
||||
|
||||
//Write MIS relation to database
|
||||
using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misdetail.db;Version=3;Journal Mode=Off;"))
|
||||
{
|
||||
sqlite.Open();
|
||||
|
||||
//Create table for MIS detail
|
||||
sqlite.Execute("CREATE TABLE MisDetail(MisFK TEXT, PartFK TEXT, CharNumber TEXT, TestGroup TEXT, TestDefinition TEXT, TestDescription TEXT, SamplingType TEXT, SamplingValue TEXT, ToolsGauges TEXT, WorkInstructions TEXT);");
|
||||
|
||||
foreach (var batch in misDetails.BatchGroup(250000))
|
||||
{
|
||||
using (SQLiteTransaction transaction = sqlite.BeginTransaction())
|
||||
{
|
||||
foreach (MisDetail misDetail in batch)
|
||||
{
|
||||
sqlite.Execute("INSERT INTO MisDetail(MisFK, PartFK, CharNumber, TestGroup, TestDefinition, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions) VALUES(@MisFK, @PartFK, @CharNumber, @TestGroup, @TestDefinition, @TestDescription, @SamplingType, @SamplingValue, @ToolsGauges, @WorkInstructions)", misDetail);
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
}
|
||||
|
||||
//Add indices on FK fields
|
||||
sqlite.Execute("CREATE INDEX IDX_MisDetail_MisFK ON MisDetail(MisFK);");
|
||||
sqlite.Execute("CREATE INDEX IDX_MisDetail_PartFK ON MisDetail(PartFK);");
|
||||
|
||||
sqlite.Execute("VACUUM");
|
||||
}
|
||||
}
|
||||
|
||||
private static void PrepareMisPart()
|
||||
{
|
||||
return;
|
||||
if (File.Exists("mispart.db"))
|
||||
{
|
||||
File.Delete("mispart.db");
|
||||
}
|
||||
SQLiteConnection.CreateFile("mispart.db");
|
||||
|
||||
IEnumerable<MisPart> misParts = CMS.GetMisParts();
|
||||
|
||||
//Write MIS relation to database
|
||||
using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=mispart.db;Version=3;Journal Mode=Off;"))
|
||||
{
|
||||
sqlite.Open();
|
||||
|
||||
//Create table for MIS detail
|
||||
sqlite.Execute("CREATE TABLE MisPart(PartFK TEXT, ItemNumber TEXT, BranchCode TEXT, SequenceNumber TEXT);");
|
||||
|
||||
foreach (var batch in misParts.BatchGroup(100000))
|
||||
{
|
||||
using (SQLiteTransaction transaction = sqlite.BeginTransaction())
|
||||
{
|
||||
foreach (MisPart misPart in batch)
|
||||
{
|
||||
sqlite.Execute("INSERT INTO MisPart(PartFK, ItemNumber, BranchCode, SequenceNumber) VALUES(@PartFK, @ItemNumber, @BranchCode, @SequenceNumber);", misPart);
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
}
|
||||
|
||||
//Add indices on FK fields
|
||||
sqlite.Execute("CREATE INDEX IDX_MisPart_PartFK ON MisPart(PartFK);");
|
||||
|
||||
sqlite.Execute("VACUUM");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+217
@@ -0,0 +1,217 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Configuration;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DataModel.Models;
|
||||
using Microsoft.AspNet.SignalR.Client;
|
||||
using NLog;
|
||||
using WorkerService.Helpers;
|
||||
using WorkerService.Models;
|
||||
using WorkerService.Models.Reporting;
|
||||
|
||||
namespace WorkerService.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// Background work processor
|
||||
/// </summary>
|
||||
public class WorkProcessor : IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Default wait/sleep interval between processing runs
|
||||
/// </summary>
|
||||
private const int WAIT_INTERVAL = 5000;
|
||||
|
||||
/// <summary>
|
||||
/// Shared logger instance
|
||||
/// </summary>
|
||||
private static readonly Logger logger = LogManager.GetLogger("WorkProcessor");
|
||||
|
||||
/// <summary>
|
||||
/// Thread cancellation flag
|
||||
/// </summary>
|
||||
private readonly ManualResetEvent cancel = new ManualResetEvent(false);
|
||||
|
||||
/// <summary>
|
||||
/// Current work status message
|
||||
/// </summary>
|
||||
public string Status
|
||||
{
|
||||
get => _status;
|
||||
set
|
||||
{
|
||||
if (string.Equals(_status, value))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_status = value;
|
||||
logger.Info("Status: {0}", _status);
|
||||
PublishStatus(_status);
|
||||
}
|
||||
}
|
||||
private string _status;
|
||||
|
||||
/// <summary>
|
||||
/// Publishes status message update to the hub
|
||||
/// </summary>
|
||||
/// <param name="status">Status message to publish</param>
|
||||
private async void PublishStatus(string status)
|
||||
{
|
||||
try
|
||||
{
|
||||
string hubHost = ConfigurationManager.AppSettings["HubHost"];
|
||||
using (HubConnection hubConnection = new HubConnection(hubHost))
|
||||
{
|
||||
IHubProxy hubProxy = hubConnection.CreateHubProxy("StatusHub");
|
||||
await hubConnection.Start();
|
||||
|
||||
StatusUpdate update = new StatusUpdate()
|
||||
{
|
||||
Message = status,
|
||||
Timestamp = DateTime.Now
|
||||
};
|
||||
await hubProxy.Invoke("SetStatus", update);
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
logger.Error("PublishStatus: failed to publish status update to hub: {0}.", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Publishes update for the search to the hub
|
||||
/// </summary>
|
||||
/// <param name="search">Search to publish update for</param>
|
||||
private async void PublishSearchUpdate(Search search)
|
||||
{
|
||||
try
|
||||
{
|
||||
string hubHost = ConfigurationManager.AppSettings["HubHost"];
|
||||
using (HubConnection hubConnection = new HubConnection(hubHost))
|
||||
{
|
||||
IHubProxy hubProxy = hubConnection.CreateHubProxy("StatusHub");
|
||||
await hubConnection.Start();
|
||||
|
||||
SearchUpdate searchUpdate = new SearchUpdate(search);
|
||||
await hubProxy.Invoke("PublishSearchUpdate", searchUpdate);
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
logger.Error("PublishSearchUpdate: failed to publish search update to hub: {0}.", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts the worker thread
|
||||
/// </summary>
|
||||
public void Start()
|
||||
{
|
||||
logger.Info("Background processing thread starting...");
|
||||
|
||||
Thread workThread = new Thread(() =>
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
DoWork();
|
||||
|
||||
//Wait to continue
|
||||
if (cancel.WaitOne(WAIT_INTERVAL))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
workThread.Start();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops the worker thread
|
||||
/// </summary>
|
||||
public void Stop()
|
||||
{
|
||||
logger.Info("Background processing thread stopping...");
|
||||
|
||||
cancel.Set();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Work processing action
|
||||
/// </summary>
|
||||
private void DoWork()
|
||||
{
|
||||
try
|
||||
{
|
||||
//Verify all data sources up to date
|
||||
List<DataUpdateTask> pending = UpdateProcessor.GetPendingUpdateTasks();
|
||||
if (pending.Any())
|
||||
{
|
||||
Status = "Updating data cache";
|
||||
Parallel.ForEach(pending, new ParallelOptions() { MaxDegreeOfParallelism = 8 }, pendingTask => { UpdateProcessor.DoUpdate(pendingTask); });
|
||||
}
|
||||
else
|
||||
{
|
||||
//Reset any partially completed searches
|
||||
LotFinderDBExt.ResetPartialSearches();
|
||||
|
||||
//Check for queued searches
|
||||
Search search = LotFinderDBExt.GetNextSearch();
|
||||
if (search != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
Status = $"Processing search #{search.ID}";
|
||||
|
||||
//Start search
|
||||
LotFinderDBExt.StartSearch(search);
|
||||
PublishSearchUpdate(search);
|
||||
|
||||
//Do search
|
||||
SearchModel searchModel = search.ToSearchModel();
|
||||
LotFinderDBExt.Search(searchModel);
|
||||
|
||||
//Record end timestamp
|
||||
search.EndDT = DateTime.Now;
|
||||
|
||||
//Generate output
|
||||
search.Results = ExcelWriter.Generate(searchModel);
|
||||
|
||||
File.WriteAllBytes($"search_{search.ID}.xlsx", search.Results);
|
||||
|
||||
//Complete search
|
||||
LotFinderDBExt.CompleteSearch(search, true);
|
||||
PublishSearchUpdate(search);
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log error and mark search as failed
|
||||
logger.Error("DoWork: failed to process search: {0}.", error.Message);
|
||||
search.EndDT = DateTime.Now;
|
||||
LotFinderDBExt.CompleteSearch(search, false);
|
||||
PublishSearchUpdate(search);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Status = "Idle";
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
//Log but do not forward error
|
||||
logger.Error("DoWork: work processing run failed: {0}.", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops worker thread
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
Stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user