From 1e5b1bb3dab8ce53d1163a440ff93c12f422c23e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 19 Jan 2026 01:08:37 -0500 Subject: [PATCH] feat(database): split MisData into archival tables Split MisData table into MisData_Curr (Status='Current') and MisData_Hist (Status='BackLevel') following existing archival pattern. - Add MisData_Curr and MisData_Hist table scripts - Create MisData view (UNION ALL) for backward compatibility - Update production and DEV pipeline configurations - Update unit tests for new table count (22 tables) - Update database documentation --- .../Pipelines/dev-pipelines.json | 14 ++++--- .../Pipelines/pipelines.json | 38 +++++++++++++++---- .../Scripts/012_CreateMisDataTable.sql | 9 +++-- .../Scripts/012a_CreateMisDataHistTable.sql | 26 +++++++++++++ .../Scripts/012b_CreateMisDataCurrTable.sql | 26 +++++++++++++ .../Scripts/033_CreateMisDataView.sql | 13 +++++++ .../DevEtlPipelineFactoryTests.cs | 11 +++--- 7 files changed, 116 insertions(+), 21 deletions(-) create mode 100644 NEW/src/JdeScoping.Database/Scripts/012a_CreateMisDataHistTable.sql create mode 100644 NEW/src/JdeScoping.Database/Scripts/012b_CreateMisDataCurrTable.sql create mode 100644 NEW/src/JdeScoping.Database/Scripts/033_CreateMisDataView.sql diff --git a/NEW/src/JdeScoping.DataSync.Dev/Pipelines/dev-pipelines.json b/NEW/src/JdeScoping.DataSync.Dev/Pipelines/dev-pipelines.json index 39ff7b3..751e24e 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/Pipelines/dev-pipelines.json +++ b/NEW/src/JdeScoping.DataSync.Dev/Pipelines/dev-pipelines.json @@ -2,8 +2,8 @@ "settings": { "sizeCategories": { "small": ["Branch", "OrgHierarchy", "WorkCenter", "ProfitCenter"], - "medium": ["JdeUser", "FunctionCode", "Item", "RouteMaster"], - "large": ["Lot", "MisData", "WorkOrder_Curr", "WorkOrder_Hist", "LotUsage_Hist", "WorkOrderComponent_Hist"], + "medium": ["JdeUser", "FunctionCode", "Item", "RouteMaster", "MisData_Curr"], + "large": ["Lot", "MisData_Hist", "WorkOrder_Curr", "WorkOrder_Hist", "LotUsage_Hist", "WorkOrderComponent_Hist"], "veryLarge": ["WorkOrderStep_Hist", "WorkOrderComponent_Curr", "WorkOrderRouting", "LotUsage_Curr", "WorkOrderStep_Curr", "WorkOrderTime_Hist", "WorkOrderTime_Curr"] } }, @@ -44,9 +44,13 @@ "source": { "fileName": "lot.pb.zstd" }, "destination": { "table": "Lot" } }, - "MisData": { - "source": { "fileName": "misdata.pb.zstd" }, - "destination": { "table": "MisData" } + "MisData_Curr": { + "source": { "fileName": "misdata_curr.pb.zstd" }, + "destination": { "table": "MisData_Curr" } + }, + "MisData_Hist": { + "source": { "fileName": "misdata_hist.pb.zstd" }, + "destination": { "table": "MisData_Hist" } }, "WorkOrder_Curr": { "source": { "fileName": "workorder_curr.pb.zstd" }, diff --git a/NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json b/NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json index c851a65..99e7187 100644 --- a/NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json +++ b/NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json @@ -173,11 +173,11 @@ "excludeFromUpdate": ["Code", "LastUpdateDt"] } }, - "MisData": { + "MisData_Curr": { "source": { "connection": "cms", - "query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME IN ('Current', 'BackLevel') AND Status.PDATE_RELEASED >= :lastUpdateDT", - "massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME IN ('Current', 'BackLevel')", + "query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current' AND Status.PDATE_RELEASED >= :lastUpdateDT", + "massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'Current'", "parameters": { "lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" } } @@ -188,13 +188,37 @@ "hourly": { "enabled": false } }, "destination": { - "table": "MisData", + "table": "MisData_Curr", "matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"] }, "postScripts": [ - "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData.MisNumber = bl.MisNumber AND MisData.RevID = bl.RevID AND MisData.Status = 'Current' AND bl.Status = 'BackLevel';", - "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData.MisNumber = nl.MisNumber AND MisData.RevID < nl.RevID AND MisData.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;", - "ALTER INDEX [PK_MisData] ON [dbo].[MisData] REBUILD;" + "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Curr.MisNumber = bl.MisNumber AND MisData_Curr.RevID = bl.RevID AND MisData_Curr.Status = 'Current' AND bl.Status = 'BackLevel';", + "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Curr AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Curr SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Curr.MisNumber = nl.MisNumber AND MisData_Curr.RevID < nl.RevID AND MisData_Curr.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;", + "ALTER INDEX [PK_MisData_Curr] ON [dbo].[MisData_Curr] REBUILD;" + ] + }, + "MisData_Hist": { + "source": { + "connection": "cms", + "query": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel' AND Status.PDATE_RELEASED >= :lastUpdateDT", + "massQuery": "SELECT DISTINCT mis.P_PART_NUMBER AS ItemNumber, mis.P_OPERATION_NUMBER AS SequenceNumber, item.PITEM_ID AS MISNumber, itemrev.PITEM_REVISION_ID AS RevID, TRIM(mis.P_SITE) AS BranchCode, zim_test_details.P_SEQ_NUMBER AS CharNumber, zim_test_details.P_TEST_DESC AS TestDescription, zim_test_details.P_SAMPL_TYPE AS SamplingType, zim_test_details.P_SAMPL_VALUE AS SamplingValue, zim_test_details.P_TOOLS AS ToolsGauges, zim_test_details.P_WORK_INTR AS WorkInstructions, Status.PNAME AS Status, Status.PDATE_RELEASED AS ReleaseDate FROM INFODBA.PITEM item INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) WHERE Status.PNAME = 'BackLevel'", + "parameters": { + "lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" } + } + }, + "schedules": { + "mass": { "intervalMinutes": 100800 }, + "daily": {}, + "hourly": { "enabled": false } + }, + "destination": { + "table": "MisData_Hist", + "matchColumns": ["ItemNumber", "BranchCode", "SequenceNumber", "MisNumber", "CharNumber"] + }, + "postScripts": [ + "SET ANSI_WARNINGS OFF; WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = bl.Released FROM cte bl WHERE MisData_Hist.MisNumber = bl.MisNumber AND MisData_Hist.RevID = bl.RevID AND MisData_Hist.Status = 'Current' AND bl.Status = 'BackLevel';", + "WITH cte AS (SELECT md.MisNumber, md.RevID, md.Status, MIN(md.ReleaseDate) Released FROM dbo.MisData_Hist AS md GROUP BY md.MisNumber, md.RevID, md.Status) UPDATE dbo.MisData_Hist SET ObsoleteDate = (SELECT TOP 1 nl.Released FROM cte nl WHERE MisData_Hist.MisNumber = nl.MisNumber AND MisData_Hist.RevID < nl.RevID AND MisData_Hist.Status = nl.Status ORDER BY nl.RevID) WHERE ObsoleteDate IS NULL;", + "ALTER INDEX [PK_MisData_Hist] ON [dbo].[MisData_Hist] REBUILD;" ] }, "WorkOrderTime_Curr": { diff --git a/NEW/src/JdeScoping.Database/Scripts/012_CreateMisDataTable.sql b/NEW/src/JdeScoping.Database/Scripts/012_CreateMisDataTable.sql index b89382c..7fe6623 100644 --- a/NEW/src/JdeScoping.Database/Scripts/012_CreateMisDataTable.sql +++ b/NEW/src/JdeScoping.Database/Scripts/012_CreateMisDataTable.sql @@ -1,10 +1,11 @@ -- Migration: 012_CreateMisDataTable -- Source: OLD/Database/Tables/MisData.sql --- Changes: DATETIME -> DATETIME2(7) +-- Changes: DATETIME -> DATETIME2(7), now creates MisData_Hist (archival pattern) +-- Note: This script is kept for fresh installs; creates the _Hist table -IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'MisData' AND schema_id = SCHEMA_ID('dbo')) +IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'MisData_Hist' AND schema_id = SCHEMA_ID('dbo')) BEGIN - CREATE TABLE [dbo].[MisData] + CREATE TABLE [dbo].[MisData_Hist] ( [ItemNumber] VARCHAR(32) NOT NULL, [BranchCode] VARCHAR(32) NOT NULL, @@ -20,7 +21,7 @@ BEGIN [Status] VARCHAR(32) NOT NULL, [ReleaseDate] DATETIME2(7) NULL, [ObsoleteDate] DATETIME2(7) NULL, - CONSTRAINT [PK_MisData] PRIMARY KEY CLUSTERED([ItemNumber], [BranchCode], [SequenceNumber], [MisNumber], [RevID], [Status], [CharNumber]) + CONSTRAINT [PK_MisData_Hist] PRIMARY KEY CLUSTERED([ItemNumber], [BranchCode], [SequenceNumber], [MisNumber], [RevID], [Status], [CharNumber]) ); END GO diff --git a/NEW/src/JdeScoping.Database/Scripts/012a_CreateMisDataHistTable.sql b/NEW/src/JdeScoping.Database/Scripts/012a_CreateMisDataHistTable.sql new file mode 100644 index 0000000..c261914 --- /dev/null +++ b/NEW/src/JdeScoping.Database/Scripts/012a_CreateMisDataHistTable.sql @@ -0,0 +1,26 @@ +-- Migration: 012a_CreateMisDataHistTable +-- Source: OLD/Database/Tables/MisData.sql +-- Changes: DATETIME -> DATETIME2(7), split into Hist/Curr tables + +IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'MisData_Hist' AND schema_id = SCHEMA_ID('dbo')) +BEGIN + CREATE TABLE [dbo].[MisData_Hist] + ( + [ItemNumber] VARCHAR(32) NOT NULL, + [BranchCode] VARCHAR(32) NOT NULL, + [SequenceNumber] VARCHAR(32) NOT NULL, + [MisNumber] VARCHAR(32) NOT NULL, + [RevID] VARCHAR(32) NOT NULL, + [CharNumber] VARCHAR(32) NOT NULL, + [TestDescription] VARCHAR(2000) NULL, + [SamplingType] VARCHAR(32) NULL, + [SamplingValue] VARCHAR(32) NULL, + [ToolsGauges] VARCHAR(2000) NULL, + [WorkInstructions] VARCHAR(2000) NULL, + [Status] VARCHAR(32) NOT NULL, + [ReleaseDate] DATETIME2(7) NULL, + [ObsoleteDate] DATETIME2(7) NULL, + CONSTRAINT [PK_MisData_Hist] PRIMARY KEY CLUSTERED([ItemNumber], [BranchCode], [SequenceNumber], [MisNumber], [RevID], [Status], [CharNumber]) + ); +END +GO diff --git a/NEW/src/JdeScoping.Database/Scripts/012b_CreateMisDataCurrTable.sql b/NEW/src/JdeScoping.Database/Scripts/012b_CreateMisDataCurrTable.sql new file mode 100644 index 0000000..3ce05c2 --- /dev/null +++ b/NEW/src/JdeScoping.Database/Scripts/012b_CreateMisDataCurrTable.sql @@ -0,0 +1,26 @@ +-- Migration: 012b_CreateMisDataCurrTable +-- Source: OLD/Database/Tables/MisData.sql +-- Changes: DATETIME -> DATETIME2(7), split into Hist/Curr tables + +IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'MisData_Curr' AND schema_id = SCHEMA_ID('dbo')) +BEGIN + CREATE TABLE [dbo].[MisData_Curr] + ( + [ItemNumber] VARCHAR(32) NOT NULL, + [BranchCode] VARCHAR(32) NOT NULL, + [SequenceNumber] VARCHAR(32) NOT NULL, + [MisNumber] VARCHAR(32) NOT NULL, + [RevID] VARCHAR(32) NOT NULL, + [CharNumber] VARCHAR(32) NOT NULL, + [TestDescription] VARCHAR(2000) NULL, + [SamplingType] VARCHAR(32) NULL, + [SamplingValue] VARCHAR(32) NULL, + [ToolsGauges] VARCHAR(2000) NULL, + [WorkInstructions] VARCHAR(2000) NULL, + [Status] VARCHAR(32) NOT NULL, + [ReleaseDate] DATETIME2(7) NULL, + [ObsoleteDate] DATETIME2(7) NULL, + CONSTRAINT [PK_MisData_Curr] PRIMARY KEY CLUSTERED([ItemNumber], [BranchCode], [SequenceNumber], [MisNumber], [RevID], [Status], [CharNumber]) + ); +END +GO diff --git a/NEW/src/JdeScoping.Database/Scripts/033_CreateMisDataView.sql b/NEW/src/JdeScoping.Database/Scripts/033_CreateMisDataView.sql new file mode 100644 index 0000000..bb60537 --- /dev/null +++ b/NEW/src/JdeScoping.Database/Scripts/033_CreateMisDataView.sql @@ -0,0 +1,13 @@ +-- Migration: 033_CreateMisDataView +-- Source: OLD/Database/Views/MisData.sql +-- Union view combining MisData_Curr and MisData_Hist + +CREATE OR ALTER VIEW [dbo].[MisData] AS +( + SELECT hist.* + FROM dbo.MisData_Hist hist + UNION ALL + SELECT curr.* + FROM dbo.MisData_Curr curr +); +GO diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/DevEtlPipelineFactoryTests.cs b/NEW/tests/JdeScoping.DataSync.Dev.Tests/DevEtlPipelineFactoryTests.cs index 5e0ccc2..5d66581 100644 --- a/NEW/tests/JdeScoping.DataSync.Dev.Tests/DevEtlPipelineFactoryTests.cs +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/DevEtlPipelineFactoryTests.cs @@ -57,7 +57,7 @@ public class DevEtlPipelineFactoryTests } [Fact] - public void GetAvailableTables_Returns21Tables() + public void GetAvailableTables_Returns22Tables() { // Arrange var factory = CreateFactoryFromConfig(); @@ -66,7 +66,7 @@ public class DevEtlPipelineFactoryTests var tables = factory.GetAvailableTables().ToList(); // Assert - tables.Count.ShouldBe(21); + tables.Count.ShouldBe(22); tables.ShouldContain("Branch"); tables.ShouldContain("WorkOrder_Curr"); tables.ShouldContain("LotUsage_Curr"); @@ -173,8 +173,8 @@ public class DevEtlPipelineFactoryTests SizeCategories = new SizeCategories { Small = ["Branch", "OrgHierarchy", "WorkCenter", "ProfitCenter"], - Medium = ["JdeUser", "FunctionCode", "Item", "RouteMaster"], - Large = ["Lot", "MisData", "WorkOrder_Curr", "WorkOrder_Hist", "LotUsage_Hist", "WorkOrderComponent_Hist"], + Medium = ["JdeUser", "FunctionCode", "Item", "RouteMaster", "MisData_Curr"], + Large = ["Lot", "MisData_Hist", "WorkOrder_Curr", "WorkOrder_Hist", "LotUsage_Hist", "WorkOrderComponent_Hist"], VeryLarge = ["WorkOrderStep_Hist", "WorkOrderComponent_Curr", "WorkOrderRouting", "LotUsage_Curr", "WorkOrderStep_Curr", "WorkOrderTime_Hist", "WorkOrderTime_Curr"] } }; @@ -190,7 +190,8 @@ public class DevEtlPipelineFactoryTests ["Item"] = new(new DevSourceConfig("item.pb.zstd"), new DevDestinationConfig("Item")), ["RouteMaster"] = new(new DevSourceConfig("routemaster.pb.zstd"), new DevDestinationConfig("RouteMaster")), ["Lot"] = new(new DevSourceConfig("lot.pb.zstd"), new DevDestinationConfig("Lot")), - ["MisData"] = new(new DevSourceConfig("misdata.pb.zstd"), new DevDestinationConfig("MisData")), + ["MisData_Curr"] = new(new DevSourceConfig("misdata_curr.pb.zstd"), new DevDestinationConfig("MisData_Curr")), + ["MisData_Hist"] = new(new DevSourceConfig("misdata_hist.pb.zstd"), new DevDestinationConfig("MisData_Hist")), ["WorkOrder_Curr"] = new(new DevSourceConfig("workorder_curr.pb.zstd"), new DevDestinationConfig("WorkOrder_Curr")), ["WorkOrder_Hist"] = new(new DevSourceConfig("workorder_hist.pb.zstd"), new DevDestinationConfig("WorkOrder_Hist")), ["LotUsage_Curr"] = new(new DevSourceConfig("lotusage_curr.pb.zstd"), new DevDestinationConfig("LotUsage_Curr")),