jdescopingtool/OLD/WorkerService/Process/Updater.MisDataPrep.cs

using System;
using System.Collections.Generic;
using System.Data.SQLite;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Commons.Helpers;
using Commons.Models;
using Commons.Process;
using Dapper;

namespace WorkerService.Process
{
    public partial class Updater
    {
        private const string SQL_GET_MIS_DATA = @"
select 
    ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, 
    CharNumber, TestGroup, TestDefinition, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions,
    Status, ReleaseDate, ObsoleteDate
FROM infodb.MisInfo info INNER JOIN
     relationdb.MisRelation rel ON (info.RevFK = rel.RevFK) INNER JOIN 
     detaildb.MisDetail det ON (rel.MisFK = det.MisFK) INNER JOIN
     partdb.MisPart part ON (det.PartFK = part.PartFK)
ORDER BY ItemNumber, BranchCode, SequenceNumber, MisNumber, RevID, Status, CharNumber";

        public static IEnumerable<MisData> GetMisData()
        {
            Task misInfoTask = Task.Run(() => PrepareMisInfo());
            Task misRelationTask = Task.Run(() => PrepareMisRelation());
            Task misDetailTask = Task.Run(() => PrepareMisDetail());
            Task misPartTask = Task.Run(() => PrepareMisPart());

            Task.WaitAll(misInfoTask, misRelationTask, misDetailTask, misPartTask);

            if (File.Exists("misdata.db"))
            {
                File.Delete("misdata.db");
            }
            SQLiteConnection.CreateFile("misdata.db");
            using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misdata.db;Version=3;Journal Mode=Off;"))
            {
                sqlite.Open();
                
                //Attach databases
                sqlite.Execute("ATTACH DATABASE 'misinfo.db' AS infodb;");
                sqlite.Execute("ATTACH DATABASE 'misrelation.db' AS relationdb;");
                sqlite.Execute("ATTACH DATABASE 'misdetail.db' AS detaildb;");
                sqlite.Execute("ATTACH DATABASE 'mispart.db' AS partdb;");

                foreach (MisData misData in sqlite.Query<MisData>(SQL_GET_MIS_DATA, buffered: false))
                {
                    yield return misData;
                }
            }
        }

        private static void PrepareMisInfo()
        {
            return;
            if (File.Exists("misinfo.db"))
            {
                File.Delete("misinfo.db");
            }
            SQLiteConnection.CreateFile("misinfo.db");

            //Fetch MIS info
            List<MisInfo> misInfos = CMS.GetMisInfos().ToList();

            //Lookup obsolete date by backlevel release
            foreach (var group in misInfos.GroupBy(m => new { m.MisNumber, m.RevID }))
            {
                DateTime? obsoleteDate = group.Where(g => g.Status.Equals("BackLevel") && g.ReleaseDate != null).Select(g => g.ReleaseDate).FirstOrDefault();

                foreach (MisInfo current in group.Where(g => g.Status.Equals("Current")))
                {
                    current.ObsoleteDate = obsoleteDate;
                }
            }

            //Lookup obsolete date by next revision release
            var lookup = misInfos.Where(m => m.ReleaseDate != null).GroupBy(m => new { m.MisNumber, m.RevID, m.Status }).Select(g => new { g.Key.MisNumber, g.Key.RevID, g.Key.Status, ReleaseDate = g.Min(m => m.ReleaseDate) }).ToDictionary(g => new { g.MisNumber, g.RevID, g.Status }, g => g.ReleaseDate);
            foreach (MisInfo misInfo in misInfos.Where(m => !m.ObsoleteDate.HasValue))
            {
                DateTime? obsoleteDate;
                if (lookup.TryGetValue(new { misInfo.MisNumber, misInfo.RevID, misInfo.Status }, out obsoleteDate))
                {
                    misInfo.ObsoleteDate = obsoleteDate;
                }
            }

            //Write MIS info to database
            using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misinfo.db;Version=3;Journal Mode=Off;"))
            {
                sqlite.Open();

                //Create table for MIS info
                sqlite.Execute("CREATE TABLE MisInfo(RevFK TEXT, MisNumber TEXT, RevID Text, Status TEXT, ReleaseDate DATETIME, ObsoleteDate DATETIME);");

                foreach (var batch in misInfos.BatchGroup(10000))
                {
                    using (SQLiteTransaction transaction = sqlite.BeginTransaction())
                    {
                        foreach (MisInfo misInfo in batch)
                        {
                            sqlite.Execute("INSERT INTO MisInfo(RevFK, MisNumber, RevID, Status, ReleaseDate, ObsoleteDate) VALUES(@RevFK, @MisNumber, @RevID, @Status, @ReleaseDate, @ObsoleteDate)", misInfo);
                        }

                        transaction.Commit();
                    }
                }

                //Add index on FK field
                sqlite.Execute("CREATE INDEX IDX_MisInfo_RevFK ON MisInfo(RevFK);");

                sqlite.Execute("VACUUM");
            }
        }

        private static void PrepareMisRelation()
        {
            return;
            if (File.Exists("misrelation.db"))
            {
                File.Delete("misrelation.db");
            }
            SQLiteConnection.CreateFile("misrelation.db");

            IEnumerable<MisRelation> misRelations = CMS.GetMisRelations();

            //Write MIS relation to database
            using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misrelation.db;Version=3;Journal Mode=Off;"))
            {
                sqlite.Open();

                //Create table for MIS relation
                sqlite.Execute("CREATE TABLE MisRelation(RevFK TEXT, MisFK TEXT);");

                foreach (var batch in misRelations.BatchGroup(10000))
                {
                    using (SQLiteTransaction transaction = sqlite.BeginTransaction())
                    {
                        foreach (MisRelation misRelation in batch)
                        {
                            sqlite.Execute("INSERT INTO MisRelation(RevFK, MisFK) VALUES(@RevFK, @MisFK)", misRelation);
                        }

                        transaction.Commit();
                    }
                }

                //Add indices on FK fields
                sqlite.Execute("CREATE INDEX IDX_MisRelation_RevFK ON MisRelation(RevFK);");
                sqlite.Execute("CREATE INDEX IDX_MisRelation_MisFK ON MisRelation(MisFK);");

                sqlite.Execute("VACUUM");
            }
        }

        public static void PrepareMisDetail()
        {
            return;
            if (File.Exists("misdetail.db"))
            {
                File.Delete("misdetail.db");
            }
            SQLiteConnection.CreateFile("misdetail.db");

            IEnumerable<MisDetail> misDetails = CMS.GetMisDetails();

            //Write MIS relation to database
            using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=misdetail.db;Version=3;Journal Mode=Off;"))
            {
                sqlite.Open();

                //Create table for MIS detail
                sqlite.Execute("CREATE TABLE MisDetail(MisFK TEXT, PartFK TEXT, CharNumber TEXT, TestGroup TEXT, TestDefinition TEXT, TestDescription TEXT, SamplingType TEXT, SamplingValue TEXT, ToolsGauges TEXT, WorkInstructions TEXT);");

                foreach (var batch in misDetails.BatchGroup(250000))
                {
                    using (SQLiteTransaction transaction = sqlite.BeginTransaction())
                    {
                        foreach (MisDetail misDetail in batch)
                        {
                            sqlite.Execute("INSERT INTO MisDetail(MisFK, PartFK, CharNumber, TestGroup, TestDefinition, TestDescription, SamplingType, SamplingValue, ToolsGauges, WorkInstructions) VALUES(@MisFK, @PartFK, @CharNumber, @TestGroup, @TestDefinition, @TestDescription, @SamplingType, @SamplingValue, @ToolsGauges, @WorkInstructions)", misDetail);
                        }

                        transaction.Commit();
                    }
                }

                //Add indices on FK fields
                sqlite.Execute("CREATE INDEX IDX_MisDetail_MisFK ON MisDetail(MisFK);");
                sqlite.Execute("CREATE INDEX IDX_MisDetail_PartFK ON MisDetail(PartFK);");

                sqlite.Execute("VACUUM");
            }
        }

        private static void PrepareMisPart()
        {
            return;
            if (File.Exists("mispart.db"))
            {
                File.Delete("mispart.db");
            }
            SQLiteConnection.CreateFile("mispart.db");

            IEnumerable<MisPart> misParts = CMS.GetMisParts();

            //Write MIS relation to database
            using (SQLiteConnection sqlite = new SQLiteConnection(@"Data Source=mispart.db;Version=3;Journal Mode=Off;"))
            {
                sqlite.Open();

                //Create table for MIS detail
                sqlite.Execute("CREATE TABLE MisPart(PartFK TEXT, ItemNumber TEXT, BranchCode TEXT, SequenceNumber TEXT);");

                foreach (var batch in misParts.BatchGroup(100000))
                {
                    using (SQLiteTransaction transaction = sqlite.BeginTransaction())
                    {
                        foreach (MisPart misPart in batch)
                        {
                            sqlite.Execute("INSERT INTO MisPart(PartFK, ItemNumber, BranchCode, SequenceNumber) VALUES(@PartFK, @ItemNumber, @BranchCode, @SequenceNumber);", misPart);
                        }

                        transaction.Commit();
                    }
                }

                //Add indices on FK fields
                sqlite.Execute("CREATE INDEX IDX_MisPart_PartFK ON MisPart(PartFK);");

                sqlite.Execute("VACUUM");
            }
        }
    }
}