using System; using System.Collections.Generic; using System.Data.SQLite; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using Commons.Helpers; using Commons.Models; using Commons.Process; using Dapper; namespace WorkerService.Process { public partial class Updater { private const string SQL_GET_MIS_DATA = @" select ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, CharNumber, TestGroup, TestDefinition, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions, Status, ReleaseDate, ObsoleteDate FROM infodb.MisInfo info INNER JOIN relationdb.MisRelation rel ON (info.RevFK = rel.RevFK) INNER JOIN detaildb.MisDetail det ON (rel.MisFK = det.MisFK) INNER JOIN partdb.MisPart part ON (det.PartFK = part.PartFK) ORDER BY ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, Status, CharNumber"; public static IEnumerable GetMisData() { Task misInfoTask = Task.Run(() => PrepareMisInfo()); Task misRelationTask = Task.Run(() => PrepareMisRelation()); Task misDetailTask = Task.Run(() => PrepareMisDetail()); Task misPartTask = Task.Run(() => PrepareMisPart()); Task.WaitAll(misInfoTask, misRelationTask, misDetailTask, misPartTask); if (File.Exists("misdata.db")) { File.Delete("misdata.db"); } SQLiteConnection.CreateFile("misdata.db"); using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misdata.db;Version=3;Journal Mode=Off;")) { sqlite.Open(); //Attach databases sqlite.Execute("ATTACH DATABASE 'misinfo.db' AS infodb;"); sqlite.Execute("ATTACH DATABASE 'misrelation.db' AS relationdb;"); sqlite.Execute("ATTACH DATABASE 'misdetail.db' AS detaildb;"); sqlite.Execute("ATTACH DATABASE 'mispart.db' AS partdb;"); foreach (MisData misData in sqlite.Query(SQL_GET_MIS_DATA, buffered: false)) { yield return misData; } } } private static void PrepareMisInfo() { return; if (File.Exists("misinfo.db")) { File.Delete("misinfo.db"); } SQLiteConnection.CreateFile("misinfo.db"); //Fetch MIS info List misInfos = CMS.GetMisInfos().ToList(); //Lookup obsolete date by backlevel release foreach (var group in misInfos.GroupBy(m => new { m.MisNumber, m.RevID })) { DateTime? obsoleteDate = group.Where(g => g.Status.Equals("BackLevel") && g.ReleaseDate != null).Select(g => g.ReleaseDate).FirstOrDefault(); foreach (MisInfo current in group.Where(g => g.Status.Equals("Current"))) { current.ObsoleteDate = obsoleteDate; } } //Lookup obsolete date by next revision release var lookup = misInfos.Where(m => m.ReleaseDate != null).GroupBy(m => new { m.MisNumber, m.RevID, m.Status }).Select(g => new { g.Key.MisNumber, g.Key.RevID, g.Key.Status, ReleaseDate = g.Min(m => m.ReleaseDate) }).ToDictionary(g => new { g.MisNumber, g.RevID, g.Status }, g => g.ReleaseDate); foreach (MisInfo misInfo in misInfos.Where(m => !m.ObsoleteDate.HasValue)) { DateTime? obsoleteDate; if (lookup.TryGetValue(new { misInfo.MisNumber, misInfo.RevID, misInfo.Status }, out obsoleteDate)) { misInfo.ObsoleteDate = obsoleteDate; } } //Write MIS info to database using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misinfo.db;Version=3;Journal Mode=Off;")) { sqlite.Open(); //Create table for MIS info sqlite.Execute("CREATE TABLE MisInfo(RevFK TEXT, MisNumber TEXT, RevID Text, Status TEXT, ReleaseDate DATETIME, ObsoleteDate DATETIME);"); foreach (var batch in misInfos.BatchGroup(10000)) { using (SQLiteTransaction transaction = sqlite.BeginTransaction()) { foreach (MisInfo misInfo in batch) { sqlite.Execute("INSERT INTO MisInfo(RevFK, MisNumber, RevID, Status, ReleaseDate, ObsoleteDate) VALUES(@RevFK, @MisNumber, @RevID, @Status, @ReleaseDate, @ObsoleteDate)", misInfo); } transaction.Commit(); } } //Add index on FK field sqlite.Execute("CREATE INDEX IDX_MisInfo_RevFK ON MisInfo(RevFK);"); sqlite.Execute("VACUUM"); } } private static void PrepareMisRelation() { return; if (File.Exists("misrelation.db")) { File.Delete("misrelation.db"); } SQLiteConnection.CreateFile("misrelation.db"); IEnumerable misRelations = CMS.GetMisRelations(); //Write MIS relation to database using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misrelation.db;Version=3;Journal Mode=Off;")) { sqlite.Open(); //Create table for MIS relation sqlite.Execute("CREATE TABLE MisRelation(RevFK TEXT, MisFK TEXT);"); foreach (var batch in misRelations.BatchGroup(10000)) { using (SQLiteTransaction transaction = sqlite.BeginTransaction()) { foreach (MisRelation misRelation in batch) { sqlite.Execute("INSERT INTO MisRelation(RevFK, MisFK) VALUES(@RevFK, @MisFK)", misRelation); } transaction.Commit(); } } //Add indices on FK fields sqlite.Execute("CREATE INDEX IDX_MisRelation_RevFK ON MisRelation(RevFK);"); sqlite.Execute("CREATE INDEX IDX_MisRelation_MisFK ON MisRelation(MisFK);"); sqlite.Execute("VACUUM"); } } public static void PrepareMisDetail() { return; if (File.Exists("misdetail.db")) { File.Delete("misdetail.db"); } SQLiteConnection.CreateFile("misdetail.db"); IEnumerable misDetails = CMS.GetMisDetails(); //Write MIS relation to database using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misdetail.db;Version=3;Journal Mode=Off;")) { sqlite.Open(); //Create table for MIS detail sqlite.Execute("CREATE TABLE MisDetail(MisFK TEXT, PartFK TEXT, CharNumber TEXT, TestGroup TEXT, TestDefinition TEXT, TestDescription TEXT, SamplingType TEXT, SamplingValue TEXT, ToolsGauges TEXT, WorkInstructions TEXT);"); foreach (var batch in misDetails.BatchGroup(250000)) { using (SQLiteTransaction transaction = sqlite.BeginTransaction()) { foreach (MisDetail misDetail in batch) { sqlite.Execute("INSERT INTO MisDetail(MisFK, PartFK, CharNumber, TestGroup, TestDefinition, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions) VALUES(@MisFK, @PartFK, @CharNumber, @TestGroup, @TestDefinition, @TestDescription, @SamplingType, @SamplingValue, @ToolsGauges, @WorkInstructions)", misDetail); } transaction.Commit(); } } //Add indices on FK fields sqlite.Execute("CREATE INDEX IDX_MisDetail_MisFK ON MisDetail(MisFK);"); sqlite.Execute("CREATE INDEX IDX_MisDetail_PartFK ON MisDetail(PartFK);"); sqlite.Execute("VACUUM"); } } private static void PrepareMisPart() { return; if (File.Exists("mispart.db")) { File.Delete("mispart.db"); } SQLiteConnection.CreateFile("mispart.db"); IEnumerable misParts = CMS.GetMisParts(); //Write MIS relation to database using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=mispart.db;Version=3;Journal Mode=Off;")) { sqlite.Open(); //Create table for MIS detail sqlite.Execute("CREATE TABLE MisPart(PartFK TEXT, ItemNumber TEXT, BranchCode TEXT, SequenceNumber TEXT);"); foreach (var batch in misParts.BatchGroup(100000)) { using (SQLiteTransaction transaction = sqlite.BeginTransaction()) { foreach (MisPart misPart in batch) { sqlite.Execute("INSERT INTO MisPart(PartFK, ItemNumber, BranchCode, SequenceNumber) VALUES(@PartFK, @ItemNumber, @BranchCode, @SequenceNumber);", misPart); } transaction.Commit(); } } //Add indices on FK fields sqlite.Execute("CREATE INDEX IDX_MisPart_PartFK ON MisPart(PartFK);"); sqlite.Execute("VACUUM"); } } } }