diff --git a/DATA_SYNC/CMS/SQL_GET_MIS_DATA.sql b/DATA_SYNC/CMS/SQL_GET_MIS_DATA.sql new file mode 100644 index 0000000..23426c0 --- /dev/null +++ b/DATA_SYNC/CMS/SQL_GET_MIS_DATA.sql @@ -0,0 +1,35 @@ +-- MisData Full Query +-- Source: CMS (Oracle via DDTek.Oracle) +-- Schema: INFODBA +-- Destination: MisData +-- Schedule: Mass/Daily (Hourly disabled) +-- Connection: Config.CMSCS +-- Timeout: Extended (1200*50 = 60000 seconds) +-- Post Processing: Commons.Process.LotFinderDB.PostProcessMisData + +SELECT DISTINCT + mis.P_PART_NUMBER AS ItemNumber, + mis.P_OPERATION_NUMBER AS SequenceNumber, + item.PITEM_ID AS MISNumber, + itemrev.PITEM_REVISION_ID AS RevID, + TRIM(mis.P_SITE) AS BranchCode, + zim_test_details.P_SEQ_NUMBER AS CharNumber, + zim_test_details.P_TEST_DESC AS TestDescription, + zim_test_details.P_SAMPL_TYPE AS SamplingType, + zim_test_details.P_SAMPL_VALUE AS SamplingValue, + zim_test_details.P_TOOLS AS ToolsGauges, + zim_test_details.P_WORK_INTR AS WorkInstructions, + Status.PNAME AS Status, + Status.PDATE_RELEASED AS ReleaseDate + FROM INFODBA.PITEM item + INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) + INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) + INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) + INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) + INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) + INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) + INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) + INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) + INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) + INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) + WHERE Status.PNAME IN ('Current', 'BackLevel') diff --git a/DATA_SYNC/CMS/SQL_GET_MIS_DATA_FILTERED.sql b/DATA_SYNC/CMS/SQL_GET_MIS_DATA_FILTERED.sql new file mode 100644 index 0000000..72af6aa --- /dev/null +++ b/DATA_SYNC/CMS/SQL_GET_MIS_DATA_FILTERED.sql @@ -0,0 +1,37 @@ +-- MisData Filtered Query (Incremental) +-- Source: CMS (Oracle via DDTek.Oracle) +-- Schema: INFODBA +-- Destination: MisData +-- Schedule: Daily (incremental merge) +-- Parameters: :lastUpdateDT (standard DateTime) +-- Connection: Config.CMSCS +-- Timeout: Extended (1200*50 = 60000 seconds) +-- Post Processing: Commons.Process.LotFinderDB.PostProcessMisData + +SELECT DISTINCT + mis.P_PART_NUMBER AS ItemNumber, + mis.P_OPERATION_NUMBER AS SequenceNumber, + item.PITEM_ID AS MISNumber, + itemrev.PITEM_REVISION_ID AS RevID, + TRIM(mis.P_SITE) AS BranchCode, + zim_test_details.P_SEQ_NUMBER AS CharNumber, + zim_test_details.P_TEST_DESC AS TestDescription, + zim_test_details.P_SAMPL_TYPE AS SamplingType, + zim_test_details.P_SAMPL_VALUE AS SamplingValue, + zim_test_details.P_TOOLS AS ToolsGauges, + zim_test_details.P_WORK_INTR AS WorkInstructions, + Status.PNAME AS Status, + Status.PDATE_RELEASED AS ReleaseDate + FROM INFODBA.PITEM item + INNER JOIN INFODBA.PITEMREVISION itemrev ON (item.PUID = itemrev.RITEMS_TAGU) + INNER JOIN INFODBA.PRELEASE_STATUS_LIST listing ON (itemrev.PUID = listing.PUID) + INNER JOIN INFODBA.PRELEASESTATUS Status ON (listing.PVALU_0 = Status.PUID) + INNER JOIN INFODBA.PIMANRELATION imanrel ON (itemrev.PUID = imanrel.RPRIMARY_OBJECTU) + INNER JOIN INFODBA.PFORM form ON (imanrel.RSECONDARY_OBJECTU = form.PUID) + INNER JOIN INFODBA.PZIMMERMISDETAILS zim_mis ON (form.RDATA_FILEU = zim_mis.PUID) + INNER JOIN INFODBA.P_TEST_DETAILS test_details ON (zim_mis.PUID = test_details.PUID) + INNER JOIN INFODBA.P_PART_ASSOCIATION ppa ON (ppa.PUID = test_details.PUID) + INNER JOIN INFODBA.PMISDATAOBJECT mis ON (mis.PUID = ppa.PVALU_0) + INNER JOIN INFODBA.PZIMTESTDETAILS zim_test_details ON (test_details.PVALU_0 = zim_test_details.PUID) + WHERE Status.PNAME IN ('Current', 'BackLevel') AND + Status.PDATE_RELEASED >= :lastUpdateDT diff --git a/DATA_SYNC/DataSyncReport.md b/DATA_SYNC/DataSyncReport.md new file mode 100644 index 0000000..0c146d7 --- /dev/null +++ b/DATA_SYNC/DataSyncReport.md @@ -0,0 +1,587 @@ +# Data Sync Configuration Report + +This document describes all data synchronization imports from the legacy JDE Scoping Tool (LotFinder) application. + +## Executive Summary + +| Source System | Database Type | Entity Count | Connection | +|---------------|---------------|--------------|------------| +| JDE | Oracle (Oracle.ManagedDataAccess) | 21 (16 active + 5 archive) | Config.JDECS | +| CMS | Oracle (DDTek.Oracle) | 1 | Config.CMSCS | + +*Note: StatusCode sync uses GIW connection (Config.GIWCS via DDTek.Oracle) instead of standard JDE connection. + +**Total Syncs:** 22 +- **Active Syncs:** 17 (scheduled to run) +- **Archive Syncs:** 5 (configured but ALL DISABLED) + +**Cache Files:** All syncs have corresponding cache files in `CACHED_DB_FILES/` (except StatusCode) + +--- + +## Schedule Legend + +| Schedule | Interval (min) | Frequency | PrepurgeData | ReIndexData | Sync Type | +|----------|----------------|-----------|--------------|-------------|-----------| +| Mass | 10080 | Weekly | Yes | Yes | Full Reload | +| Daily | 1440 | Daily | No | No | Incremental Merge | +| Hourly | 60 | Hourly | No | No | Incremental Merge | + +- **PrepurgeData=true**: Table is truncated before import (full reload) +- **PrepurgeData=false**: Records are merged/upserted (incremental) +- **Filtered Query**: Uses date/time parameters to fetch only changed records + +--- + +## Master Sync Table + +### Active Syncs (17) + +| # | Import Name | Source | Dest Table | Mass | Daily | Hourly | Filter | Cache File | Notes | +|---|-------------|--------|------------|------|-------|--------|--------|------------|-------| +| 1 | WorkOrder | JDE | WorkOrder_Curr | Yes | Yes | Yes | Yes | `workorder_curr.json.zstd` | | +| 2 | LotUsage | JDE | LotUsage_Curr | Yes | Yes | Yes | Yes | `lotusage_curr.json.zstd` | | +| 3 | Item | JDE | Item | Yes | Yes | Yes | Yes | `item.json.zstd` | | +| 4 | Lot | JDE | Lot | Yes | Yes | Yes | Yes | `lot.json.zstd` | | +| 5 | WorkOrderTime | JDE | WorkOrderTime_Curr | Yes | Yes | Yes | Yes | `workordertime_curr.json.zstd` | | +| 6 | WorkOrderComponent | JDE | WorkOrderComponent_Curr | Yes | Yes | Yes | Yes | `workordercomponent_curr.json.zstd` | | +| 7 | WorkOrderStep | JDE | WorkOrderStep_Curr | Yes | Yes | Yes | Yes | `workorderstep_curr.json.zstd` | | +| 8 | WorkOrderRouting | JDE | WorkOrderRouting | Yes | Yes | Yes | Yes | `workorderrouting.json.zstd` | | +| 9 | Branch | JDE | Branch | Yes | Yes | Yes | Yes | `branch.json.zstd` | typeCode='BP' | +| 10 | ProfitCenter | JDE | ProfitCenter | Yes | Yes | Yes | Yes | `profitcenter.json.zstd` | typeCode='I3' | +| 11 | WorkCenter | JDE | WorkCenter | Yes | Yes | Yes | Yes | `workcenter.json.zstd` | typeCode='WC' | +| 12 | StatusCode | JDE | StatusCode | Yes | Yes | Yes | Yes | *(none)* | GIW connection | +| 13 | JdeUser | JDE | JdeUser | Yes | Yes | Yes | No | `jdeuser.json.zstd` | Same query both | +| 14 | OrgHierarchy | JDE | OrgHierarchy | Yes | Yes | Yes | Yes | `orghierarchy.json.zstd` | | +| 15 | RouteMaster | JDE | RouteMaster | Yes | Yes | Yes | Yes | `routemaster.json.zstd` | | +| 16 | FunctionCode | JDE | FunctionCode | Yes | Yes | Yes | No | `functioncode.json.zstd` | Always full reload | +| 17 | MisData | CMS | MisData | Yes | Yes | No | Yes | `misdata.json.zstd` | Hourly disabled | + +### Archive Syncs (5) - ALL DISABLED + +| # | Import Name | Source | Dest Table | Mass | Daily | Hourly | Filter | Cache File | Notes | +|---|-------------|--------|------------|------|-------|--------|--------|------------|-------| +| 18 | WorkOrder_Archive | JDE | WorkOrder_Hist | No | No | No | No | `workorder_hist.json.zstd` | DISABLED | +| 19 | LotUsage_Archive | JDE | LotUsage_Hist | No | No | No | No | `lotusage_hist.json.zstd` | DISABLED | +| 20 | WorkOrderTime_Archive | JDE | WorkOrderTime_Hist | No | No | No | No | `workordertime_hist.json.zstd` | DISABLED | +| 21 | WorkOrderComponent_Archive | JDE | WorkOrderComponent_Hist | No | No | No | No | `workordercomponent_hist.json.zstd` | DISABLED | +| 22 | WorkOrderStep_Archive | JDE | WorkOrderStep_Hist | No | No | No | No | `workorderstep_hist.json.zstd` | DISABLED | + +**Cache File Location:** `CACHED_DB_FILES/` + +--- + +## Detailed Entity Configurations + +### 1. WorkOrder + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | WORKORDER | +| Destination Table | WorkOrder_Curr | +| Data Fetch Function | Commons.Process.JDE.GetWorkOrders | +| Post Processing | None | +| JDE Table | JDESTAGE.F4801_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_WORKORDERS.sql` +- Filtered: `JDE/SQL_GET_WORKORDERS_FILTERED.sql` +- Archive: `JDE_ARCHIVE/SQL_GET_WORKORDERS.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +--- + +### 2. LotUsage + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | LOTUSAGE | +| Destination Table | LotUsage_Curr | +| Data Fetch Function | Commons.Process.JDE.GetLotUsages | +| Post Processing | None | +| JDE Table | JDESTAGE.F4111_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_LOT_USAGES.sql` +- Filtered: `JDE/SQL_GET_LOT_USAGES_FILTERED.sql` +- Archive: `JDE_ARCHIVE/SQL_GET_LOT_USAGES.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +--- + +### 3. Item + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | ITEM | +| Destination Table | Item | +| Data Fetch Function | Commons.Process.JDE.GetItems | +| Post Processing | None | +| JDE Table | JDESTAGE.F4101_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_ITEMS.sql` +- Filtered: `JDE/SQL_GET_ITEMS_FILTERED.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +--- + +### 4. Lot + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | LOT | +| Destination Table | Lot | +| Data Fetch Function | Commons.Process.JDE.GetLots | +| Post Processing | None | +| JDE Table | JDESTAGE.F4108_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_LOTS.sql` +- Filtered: `JDE/SQL_GET_LOTS_FILTERED.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +--- + +### 5. WorkOrderTime + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | WORKORDERTIME | +| Destination Table | WorkOrderTime_Curr | +| Data Fetch Function | Commons.Process.JDE.GetWorkOrderTimes | +| Post Processing | None | +| JDE Table | JDESTAGE.F31122_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_WORKORDER_TIMES.sql` +- Filtered: `JDE/SQL_GET_WORKORDER_TIMES_FILTERED.sql` +- Archive: `JDE_ARCHIVE/SQL_GET_WORKORDER_TIMES.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +--- + +### 6. WorkOrderComponent + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | WORKORDERCOMPONENT | +| Destination Table | WorkOrderComponent_Curr | +| Data Fetch Function | Commons.Process.JDE.GetWorkOrderComponents | +| Post Processing | None | +| JDE Table | JDESTAGE.F3111_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_WORKORDER_COMPONENTS.sql` +- Filtered: `JDE/SQL_GET_WORKORDER_COMPONENTS_FILTERED.sql` +- Archive: `JDE_ARCHIVE/SQL_GET_WORKORDER_COMPONENTS.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +--- + +### 7. WorkOrderStep + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | WORKORDERSTEP | +| Destination Table | WorkOrderStep_Curr | +| Data Fetch Function | Commons.Process.JDE.GetWorkOrderSteps | +| Post Processing | None | +| JDE Table | JDESTAGE.F3112_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_WORKORDER_STEP.sql` +- Filtered: `JDE/SQL_GET_WORKORDER_STEP_FILTERED.sql` +- Archive: `JDE_ARCHIVE/SQL_GET_WORKORDER_STEP.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +--- + +### 8. WorkOrderRouting + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | WORKORDERROUTING | +| Destination Table | WorkOrderRouting | +| Data Fetch Function | Commons.Process.JDE.GetWorkOrderRoutings | +| Post Processing | None | +| JDE Table | JDESTAGE.F3112Z1_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_WORKORDER_ROUTING.sql` +- Filtered: `JDE/SQL_GET_WORKORDER_ROUTING_FILTERED.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +**Special Processing:** Filters out records with invalid dates (year < 1900 or > 2500) + +--- + +### 9. Branch + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | BRANCH | +| Destination Table | Branch | +| Data Fetch Function | Commons.Process.JDE.GetBranches | +| Post Processing | None | +| JDE Table | JDESTAGE.F0006_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_BUSINESS_UNITS.sql` +- Filtered: `JDE/SQL_GET_BUSINESS_UNITS_FILTERED.sql` + +**Filter Parameters:** `typeCode='BP'`, `dateUpdated`, `timeUpdated` + +--- + +### 10. ProfitCenter + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | PROFITCENTER | +| Destination Table | ProfitCenter | +| Data Fetch Function | Commons.Process.JDE.GetProfitCenters | +| Post Processing | None | +| JDE Table | JDESTAGE.F0006_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_BUSINESS_UNITS.sql` +- Filtered: `JDE/SQL_GET_BUSINESS_UNITS_FILTERED.sql` + +**Filter Parameters:** `typeCode='I3'`, `dateUpdated`, `timeUpdated` + +--- + +### 11. WorkCenter + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | WORKCENTER | +| Destination Table | WorkCenter | +| Data Fetch Function | Commons.Process.JDE.GetWorkCenters | +| Post Processing | None | +| JDE Table | JDESTAGE.F0006_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_BUSINESS_UNITS.sql` +- Filtered: `JDE/SQL_GET_BUSINESS_UNITS_FILTERED.sql` + +**Filter Parameters:** `typeCode='WC'`, `dateUpdated`, `timeUpdated` + +--- + +### 12. StatusCode + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | STATUSCODE | +| Destination Table | StatusCode | +| Data Fetch Function | Commons.Process.JDE.GetStatusCodes | +| Post Processing | None | +| JDE Table | JDESTAGE.F0005_VIEW | +| **Connection** | **GIW (not JDE)** | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_STATUS_CODES.sql` +- Filtered: `JDE/SQL_GET_STATUS_CODES_FILTERED.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` + +**Note:** This sync uses the GIW connection (Config.GIWCS) via DDTek.Oracle, not the standard JDE connection. + +--- + +### 13. JdeUser + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | USER | +| Destination Table | JdeUser | +| Data Fetch Function | Commons.Process.JDE.GetUsers | +| Post Processing | None | +| JDE Tables | JDESTAGE.F0101_VIEW, JDESTAGE.F0092_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_USERS.sql` + +**Note:** No filtered query variant exists. The same query is used for both full and incremental syncs (filter parameters are passed but not used in query). + +--- + +### 14. OrgHierarchy + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | ORGHIERARCHY | +| Destination Table | OrgHierarchy | +| Data Fetch Function | Commons.Process.JDE.GetOrgHierarchy | +| Post Processing | None | +| JDE Table | JDESTAGE.F30006_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_ORG_HIERARCHY.sql` +- Filtered: `JDE/SQL_GET_ORG_HIERARCHY_FILTERED.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +--- + +### 15. RouteMaster + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | ROUTEMASTER | +| Destination Table | RouteMaster | +| Data Fetch Function | Commons.Process.JDE.GetRouteMasters | +| Post Processing | None | +| JDE Table | JDESTAGE.F3003_VIEW | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | Yes | 60 | No | No | + +**Query Files:** +- Full: `JDE/SQL_GET_ROUTE_MASTER.sql` +- Filtered: `JDE/SQL_GET_ROUTE_MASTER_FILTERED.sql` + +**Filter Parameters:** `dateUpdated`, `timeUpdated` (JDE date/time format) + +--- + +### 16. FunctionCode + +| Property | Value | +|----------|-------| +| Source System | JDE | +| Source Data | FUNCTIONCODE | +| Destination Table | FunctionCode | +| Data Fetch Function | Commons.Process.JDE.GetFunctionCodes | +| Post Processing | None | +| JDE Table | PRODDTA.F00192 (direct table, not view) | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 10080 | Yes | Yes | +| Daily | Yes | 1440 | Yes | Yes | +| Hourly | Yes | 60 | Yes | Yes | + +**Query Files:** +- Full: `JDE/SQL_GET_FUNCTION_CODES.sql` + +**Note:** No filtered query variant. Always performs full reload (PrepurgeData=true for all schedules). Uses aggregation with LISTAGG to combine multi-row descriptions. + +--- + +### 17. MisData + +| Property | Value | +|----------|-------| +| Source System | CMS | +| Source Data | MISDATA | +| Destination Table | MisData | +| Data Fetch Function | Commons.Process.CMS.GetMisData | +| Post Processing | Commons.Process.LotFinderDB.PostProcessMisData | +| CMS Schema | INFODBA | + +**Schedule Configuration:** +| Schedule | Enabled | Interval | Prepurge | ReIndex | +|----------|---------|----------|----------|---------| +| Mass | Yes | 100800 | Yes | Yes | +| Daily | Yes | 1440 | No | No | +| Hourly | **No** | 60 | No | No | + +**Query Files:** +- Full: `CMS/SQL_GET_MIS_DATA.sql` +- Filtered: `CMS/SQL_GET_MIS_DATA_FILTERED.sql` + +**Filter Parameters:** `lastUpdateDT` (standard DateTime) + +**Note:** +- Mass interval is 100800 minutes (~10 weeks), much longer than other syncs +- Hourly sync is disabled +- Has post-processing action for additional data transformation +- Query timeout is extended (1200*50 = 60000 seconds) +- ReleaseDate is converted to local time after fetch + +--- + +## Archive Query Pattern + +Archive syncs (#18-22) fetch historical data using UNION ALL from both current and archived schemas: + +```sql +SELECT ... FROM QADTA.F{table} +UNION ALL +SELECT ... FROM ARCDTAQA.F{table} +``` + +**Query Files:** Located in `JDE_ARCHIVE/` folder. + +**Note:** Archive syncs have `IsEnabled=true` in dsconfig but all schedule types have `Enabled=false`. They are configured for manual/on-demand execution only. + +--- + +## Source Database Reference + +### JDE Oracle Connection (Config.JDECS) +- **Driver:** Oracle.ManagedDataAccess.Client +- **Schemas Used:** + - `JDESTAGE` - Views for current production data + - `PRODDTA` - Direct table access (FunctionCode only) + - `QADTA` - Current data for archive queries + - `ARCDTAQA` - Archived historical data + +### GIW Oracle Connection (Config.GIWCS) +- **Driver:** DDTek.Oracle +- **Used By:** StatusCode sync only +- **Schema:** JDESTAGE + +### CMS Oracle Connection (Config.CMSCS) +- **Driver:** DDTek.Oracle +- **Schema:** INFODBA +- **Tables:** Complex 11-table join for MIS data + +### JDE Date/Time Format +JDE uses a special date/time format conversion: +- Dates are converted via `ToJDEDate()` helper +- Times are converted via `ToJDETime()` helper +- Filter conditions use: `(date > :dateUpdated) OR (date = :dateUpdated AND time >= :timeUpdated)` + +--- + +## Source Files Reference + +| File Type | Location | +|-----------|----------| +| dsconfig JSON files | OLD/WorkerService/dsconfig/*.json | +| JDE Source Queries | OLD/WorkerService/bin/UPDATER/JdeQueries/*.sql | +| JDE Archive Queries | OLD/WorkerService/bin/UPDATER/JdeArchivalQueries/*.sql | +| CMS Source Queries | OLD/DataModel/CmsQueries/*.sql | +| C# Data Fetch Code | OLD/DataModel/Process/JDE.*.cs, CMS.*.cs | +| Configuration | OLD/DataModel/Config.cs | +| Query Repository | OLD/DataModel/Process/QueryRepository.cs | diff --git a/DATA_SYNC/JDE/SQL_GET_BUSINESS_UNITS.sql b/DATA_SYNC/JDE/SQL_GET_BUSINESS_UNITS.sql new file mode 100644 index 0000000..075fb50 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_BUSINESS_UNITS.sql @@ -0,0 +1,12 @@ +-- Business Units Full Query (Branch, ProfitCenter, WorkCenter) +-- Source: JDESTAGE.F0006_VIEW +-- Destination: Branch (typeCode='BP'), ProfitCenter (typeCode='I3'), WorkCenter (typeCode='WC') +-- Schedule: Mass/Daily/Hourly +-- Parameter: :typeCode ('BP', 'I3', or 'WC') + +SELECT TRIM(wc.COSTCENTER_MCMCU) AS Code, + TRIM(wc.DESCRIPTION001_MCDL01) AS Description, + wc.DATEUPDATED_MCUPMJ AS DateUpdated, + wc.TIMELASTUPDATED_MCUPMT AS TimeUpdated +FROM JDESTAGE.F0006_VIEW wc +WHERE wc.COSTCENTERTYPE_MCSTYL = :typeCode diff --git a/DATA_SYNC/JDE/SQL_GET_BUSINESS_UNITS_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_BUSINESS_UNITS_FILTERED.sql new file mode 100644 index 0000000..fa653c5 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_BUSINESS_UNITS_FILTERED.sql @@ -0,0 +1,16 @@ +-- Business Units Filtered Query (Branch, ProfitCenter, WorkCenter) +-- Source: JDESTAGE.F0006_VIEW +-- Destination: Branch (typeCode='BP'), ProfitCenter (typeCode='I3'), WorkCenter (typeCode='WC') +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :typeCode ('BP', 'I3', or 'WC'), :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT TRIM(wc.COSTCENTER_MCMCU) AS Code, + TRIM(wc.DESCRIPTION001_MCDL01) AS Description, + wc.DATEUPDATED_MCUPMJ AS DateUpdated, + wc.TIMELASTUPDATED_MCUPMT AS TimeUpdated +FROM JDESTAGE.F0006_VIEW wc +WHERE wc.COSTCENTERTYPE_MCSTYL = :typeCode AND + ( + wc.DATEUPDATED_MCUPMJ > :dateUpdated OR + (wc.DATEUPDATED_MCUPMJ = :dateUpdated AND wc.TIMELASTUPDATED_MCUPMT >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_FUNCTION_CODES.sql b/DATA_SYNC/JDE/SQL_GET_FUNCTION_CODES.sql new file mode 100644 index 0000000..1ea90b1 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_FUNCTION_CODES.sql @@ -0,0 +1,21 @@ +-- FunctionCode Query (Full only - always full reload) +-- Source: PRODDTA.F00192 (direct table, not view) +-- Destination: FunctionCode +-- Schedule: Mass/Daily/Hourly (all with PrepurgeData=true) +-- Note: No filtered variant exists. Always performs full reload. +-- Note: Uses LISTAGG to combine multi-row descriptions into single row per code. + +SELECT Code, + TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, + SYSDATE AS LastUpdateDT +FROM ( + SELECT TRIM(fc.CFKY) AS Code, + TRIM(ASCIISTR(fc.CFDS80)) AS Description, + SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, + SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, + COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values + FROM PRODDTA.F00192 fc + WHERE TRIM(fc.CFKY) IS NOT NULL +) +WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') +GROUP BY Code diff --git a/DATA_SYNC/JDE/SQL_GET_ITEMS.sql b/DATA_SYNC/JDE/SQL_GET_ITEMS.sql new file mode 100644 index 0000000..8d3acfa --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_ITEMS.sql @@ -0,0 +1,13 @@ +-- Item Full Query +-- Source: JDESTAGE.F4101_VIEW +-- Destination: Item +-- Schedule: Mass/Daily/Hourly + +SELECT pn.IDENTIFIERSHORTITEM_IMITM AS ShortItemNumber, + TRIM(pn.IDENTIFIER2NDITEM_IMLITM) AS ItemNumber, + TRIM(pn.DESCRIPTIONLINE1_IMDSC1) AS Description, + TRIM(pn.PURCHASINGREPORTCODE4_IMPRP4) AS PlanningFamily, + pn.DATEUPDATED_IMUPMJ AS DateUpdated, + pn.TIMEOFDAY_IMTDAY AS TimeUpdated + FROM JDESTAGE.F4101_VIEW pn + WHERE TRIM(pn.IDENTIFIER2NDITEM_IMLITM) IS NOT NULL diff --git a/DATA_SYNC/JDE/SQL_GET_ITEMS_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_ITEMS_FILTERED.sql new file mode 100644 index 0000000..f7234f8 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_ITEMS_FILTERED.sql @@ -0,0 +1,18 @@ +-- Item Filtered Query (Incremental) +-- Source: JDESTAGE.F4101_VIEW +-- Destination: Item +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT pn.IDENTIFIERSHORTITEM_IMITM AS ShortItemNumber, + TRIM(pn.IDENTIFIER2NDITEM_IMLITM) AS ItemNumber, + TRIM(pn.DESCRIPTIONLINE1_IMDSC1) AS Description, + TRIM(pn.PURCHASINGREPORTCODE4_IMPRP4) AS PlanningFamily, + pn.DATEUPDATED_IMUPMJ AS DateUpdated, + pn.TIMEOFDAY_IMTDAY AS TimeUpdated + FROM JDESTAGE.F4101_VIEW pn + WHERE TRIM(pn.IDENTIFIER2NDITEM_IMLITM) IS NOT NULL AND + ( + pn.DATEUPDATED_IMUPMJ > :dateUpdated OR + (pn.DATEUPDATED_IMUPMJ = :dateUpdated AND pn.TIMEOFDAY_IMTDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_LOTS.sql b/DATA_SYNC/JDE/SQL_GET_LOTS.sql new file mode 100644 index 0000000..ba96f89 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_LOTS.sql @@ -0,0 +1,15 @@ +-- Lot Full Query +-- Source: JDESTAGE.F4108_VIEW +-- Destination: Lot +-- Schedule: Mass/Daily/Hourly + +SELECT TRIM(lot.LOT_IOLOTN) AS LotNumber, + TRIM(lot.COSTCENTER_IOMCU) AS BranchCode, + lot.IDENTIFIERSHORTITEM_IOITM AS ShortItemNumber, + TRIM(lot.IDENTIFIER2NDITEM_IOLITM) AS ItemNumber, + lot.PRIMARYLASTVENDORNO_IOVEND AS SupplierCode, + lot.DATEUPDATED_IOUPMJ AS DateUpdated, + lot.TIMEOFDAY_IOTDAY AS TimeUpdated + FROM JDESTAGE.F4108_VIEW lot + WHERE TRIM(lot.LOT_IOLOTN) IS NOT NULL AND + TRIM(lot.COSTCENTER_IOMCU) IS NOT NULL diff --git a/DATA_SYNC/JDE/SQL_GET_LOTS_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_LOTS_FILTERED.sql new file mode 100644 index 0000000..3eb2199 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_LOTS_FILTERED.sql @@ -0,0 +1,20 @@ +-- Lot Filtered Query (Incremental) +-- Source: JDESTAGE.F4108_VIEW +-- Destination: Lot +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT TRIM(lot.LOT_IOLOTN) AS LotNumber, + TRIM(lot.COSTCENTER_IOMCU) AS BranchCode, + lot.IDENTIFIERSHORTITEM_IOITM AS ShortItemNumber, + TRIM(lot.IDENTIFIER2NDITEM_IOLITM) AS ItemNumber, + lot.PRIMARYLASTVENDORNO_IOVEND AS SupplierCode, + lot.DATEUPDATED_IOUPMJ AS DateUpdated, + lot.TIMEOFDAY_IOTDAY AS TimeUpdated + FROM JDESTAGE.F4108_VIEW lot + WHERE TRIM(lot.LOT_IOLOTN) IS NOT NULL AND + TRIM(lot.COSTCENTER_IOMCU) IS NOT NULL AND + ( + lot.DATEUPDATED_IOUPMJ > :dateUpdated OR + (lot.DATEUPDATED_IOUPMJ = :dateUpdated AND lot.TIMEOFDAY_IOTDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_LOT_USAGES.sql b/DATA_SYNC/JDE/SQL_GET_LOT_USAGES.sql new file mode 100644 index 0000000..2549396 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_LOT_USAGES.sql @@ -0,0 +1,16 @@ +-- LotUsage Full Query +-- Source: JDESTAGE.F4111_VIEW +-- Destination: LotUsage_Curr +-- Schedule: Mass/Daily/Hourly + +SELECT lu.UNIQUEKEYIDINTERNAL_ILUKID AS UniqueID, + lu.DOCUMENTORDERINVOICEE_ILDOCO AS WorkOrderNumber, + TRIM(lu.LOT_ILLOTN) AS LotNumber, + TRIM(lu.COSTCENTER_ILMCU) AS BranchCode, + lu.IDENTIFIERSHORTITEM_ILITM AS ShortItemNumber, + lu.QUANTITYTRANSACTION_ILTRQT AS Quantity, + lu.DATETRANSACTIONJULIAN_ILTRDJ AS DateUpdated, + lu.TIMEOFDAY_ILTDAY AS TimeUpdated + FROM JDESTAGE.F4111_VIEW lu + WHERE lu.DOCUMENTTYPE_ILDCT = 'IM' AND + TRIM(lu.LOT_ILLOTN) IS NOT NULL diff --git a/DATA_SYNC/JDE/SQL_GET_LOT_USAGES_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_LOT_USAGES_FILTERED.sql new file mode 100644 index 0000000..3d30b66 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_LOT_USAGES_FILTERED.sql @@ -0,0 +1,21 @@ +-- LotUsage Filtered Query (Incremental) +-- Source: JDESTAGE.F4111_VIEW +-- Destination: LotUsage_Curr +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT lu.UNIQUEKEYIDINTERNAL_ILUKID AS UniqueID, + lu.DOCUMENTORDERINVOICEE_ILDOCO AS WorkOrderNumber, + TRIM(lu.LOT_ILLOTN) AS LotNumber, + TRIM(lu.COSTCENTER_ILMCU) AS BranchCode, + lu.IDENTIFIERSHORTITEM_ILITM AS ShortItemNumber, + lu.QUANTITYTRANSACTION_ILTRQT AS Quantity, + lu.DATETRANSACTIONJULIAN_ILTRDJ AS DateUpdated, + lu.TIMEOFDAY_ILTDAY AS TimeUpdated + FROM JDESTAGE.F4111_VIEW lu + WHERE lu.DOCUMENTTYPE_ILDCT = 'IM' AND + TRIM(lu.LOT_ILLOTN) IS NOT NULL AND + ( + lu.DATETRANSACTIONJULIAN_ILTRDJ > :dateUpdated OR + (lu.DATETRANSACTIONJULIAN_ILTRDJ = :dateUpdated AND lu.TIMEOFDAY_ILTDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_ORG_HIERARCHY.sql b/DATA_SYNC/JDE/SQL_GET_ORG_HIERARCHY.sql new file mode 100644 index 0000000..3486793 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_ORG_HIERARCHY.sql @@ -0,0 +1,13 @@ +-- OrgHierarchy Full Query +-- Source: JDESTAGE.F30006_VIEW +-- Destination: OrgHierarchy +-- Schedule: Mass/Daily/Hourly + +SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, + TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, + TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, + oh.DATEUPDATED_IWUPMJ AS DateUpdated, + oh.TIMEOFDAY_IWTDAY AS TimeUpdated + FROM JDESTAGE.F30006_VIEW oh + WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND + TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL diff --git a/DATA_SYNC/JDE/SQL_GET_ORG_HIERARCHY_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_ORG_HIERARCHY_FILTERED.sql new file mode 100644 index 0000000..7970388 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_ORG_HIERARCHY_FILTERED.sql @@ -0,0 +1,18 @@ +-- OrgHierarchy Filtered Query (Incremental) +-- Source: JDESTAGE.F30006_VIEW +-- Destination: OrgHierarchy +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT TRIM(oh.DISPATCHGROUP_IWMCUW) AS ProfitCenterCode, + TRIM(oh.COSTCENTER_IWMCU) AS WorkCenterCode, + TRIM(oh.COSTCENTERALT_IWMMCU) AS BranchCode, + oh.DATEUPDATED_IWUPMJ AS DateUpdated, + oh.TIMEOFDAY_IWTDAY AS TimeUpdated + FROM JDESTAGE.F30006_VIEW oh + WHERE TRIM(oh.COSTCENTER_IWMCU) IS NOT NULL AND + TRIM(oh.COSTCENTERALT_IWMMCU) IS NOT NULL AND + ( + oh.DATEUPDATED_IWUPMJ > :dateUpdated OR + (oh.DATEUPDATED_IWUPMJ = :dateUpdated AND oh.TIMEOFDAY_IWTDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_ROUTE_MASTER.sql b/DATA_SYNC/JDE/SQL_GET_ROUTE_MASTER.sql new file mode 100644 index 0000000..64b61ed --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_ROUTE_MASTER.sql @@ -0,0 +1,17 @@ +-- RouteMaster Full Query +-- Source: JDESTAGE.F3003_VIEW +-- Destination: RouteMaster +-- Schedule: Mass/Daily/Hourly + +SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, + TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, + TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, + route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, + TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, + TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, + route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, + route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, + route_master.DATEUPDATED_IRUPMJ AS DateUpdated, + route_master.TIMEOFDAY_IRTDAY AS TimeUpdated + FROM JDESTAGE.F3003_VIEW route_master + WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL diff --git a/DATA_SYNC/JDE/SQL_GET_ROUTE_MASTER_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_ROUTE_MASTER_FILTERED.sql new file mode 100644 index 0000000..7b3beee --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_ROUTE_MASTER_FILTERED.sql @@ -0,0 +1,22 @@ +-- RouteMaster Filtered Query (Incremental) +-- Source: JDESTAGE.F3003_VIEW +-- Destination: RouteMaster +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT TRIM(route_master.COSTCENTERALT_IRMMCU) AS BranchCode, + TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) AS ItemNumber, + TRIM(route_master.TYPEROUTING_IRTRT) AS RoutingType, + route_master.SEQUENCENOOPERATIONS_IROPSQ AS SequenceNumber, + TRIM(route_master.USERRESERVEDREFERENCE_IRURRF) AS FunctionCode, + TRIM(route_master.COSTCENTER_IRMCU) AS WorkCenterCode, + route_master.EFFECTIVEFROMDATE_IREFFF AS StartDate, + route_master.EFFECTIVETHRUDATE_IREFFT AS EndDate, + route_master.DATEUPDATED_IRUPMJ AS DateUpdated, + route_master.TIMEOFDAY_IRTDAY AS TimeUpdated + FROM JDESTAGE.F3003_VIEW route_master + WHERE TRIM(route_master.ITEMNUMBER2NDKIT_IRKITL) IS NOT NULL AND + ( + route_master.DATEUPDATED_IRUPMJ > :dateUpdated OR + (route_master.DATEUPDATED_IRUPMJ = :dateUpdated AND route_master.TIMEOFDAY_IRTDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_STATUS_CODES.sql b/DATA_SYNC/JDE/SQL_GET_STATUS_CODES.sql new file mode 100644 index 0000000..b8883a4 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_STATUS_CODES.sql @@ -0,0 +1,14 @@ +-- StatusCode Full Query +-- Source: JDESTAGE.F0005_VIEW +-- Destination: StatusCode +-- Schedule: Mass/Daily/Hourly +-- Note: Uses GIW connection (Config.GIWCS), not JDE connection + +SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS CODE, + TRIM(sc.DESCRIPTION001_DRDL01) AS Description, + sc.DATEUPDATED_DRUPMJ AS DateUpdated, + sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated + FROM JDESTAGE.F0005_VIEW sc + WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND + sc.USERDEFINEDCODES_DRRT = 'SS' AND + TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL diff --git a/DATA_SYNC/JDE/SQL_GET_STATUS_CODES_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_STATUS_CODES_FILTERED.sql new file mode 100644 index 0000000..d23c178 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_STATUS_CODES_FILTERED.sql @@ -0,0 +1,19 @@ +-- StatusCode Filtered Query (Incremental) +-- Source: JDESTAGE.F0005_VIEW +-- Destination: StatusCode +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated, :timeUpdated +-- Note: Uses GIW connection (Config.GIWCS), not JDE connection + +SELECT TRIM(sc.USERDEFINEDCODE_DRKY) AS CODE, + TRIM(sc.DESCRIPTION001_DRDL01) AS Description, + sc.DATEUPDATED_DRUPMJ AS DateUpdated, + sc.TIMELASTUPDATED_DRUPMT AS TimeUpdated + FROM JDESTAGE.F0005_VIEW sc + WHERE TRIM(sc.PRODUCTCODE_DRSY) = '00' AND + sc.USERDEFINEDCODES_DRRT = 'SS' AND + TRIM(sc.USERDEFINEDCODE_DRKY) IS NOT NULL AND + ( + sc.DATEUPDATED_DRUPMJ > :dateUpdated OR + (sc.DATEUPDATED_DRUPMJ = :dateUpdated AND sc.TIMELASTUPDATED_DRUPMT >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_USERS.sql b/DATA_SYNC/JDE/SQL_GET_USERS.sql new file mode 100644 index 0000000..6a89d91 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_USERS.sql @@ -0,0 +1,24 @@ +-- JdeUser Query (Full only - no filtered variant) +-- Source: JDESTAGE.F0101_VIEW, JDESTAGE.F0092_VIEW +-- Destination: JdeUser +-- Schedule: Mass/Daily/Hourly +-- Note: Same query used for both full and incremental (filter params passed but not used) + +WITH USER_CTE AS ( + SELECT ab.ADDRESSNUMBER_ABAN8 AS AddressNumber, + TRIM(pro.USERID_ULUSER) AS UserID, + TRIM(ab.NAMEALPHA_ABALPH) AS FullName, + ab.DATEUPDATED_ABUPMJ AS DateUpdated, + ab.TIMELASTUPDATED_ABUPMT AS TimeUpdated, + ROW_NUMBER() OVER (PARTITION BY ab.ADDRESSNUMBER_ABAN8 ORDER BY ab.DATEUPDATED_ABUPMJ DESC, ab.TIMELASTUPDATED_ABUPMT DESC) RN + FROM JDESTAGE.F0101_VIEW ab LEFT OUTER JOIN + JDESTAGE.F0092_VIEW pro ON (ab.ADDRESSNUMBER_ABAN8 = pro.ADDRESSNUMBER_ULAN8) + WHERE ab.ADDRESSTYPEEMPLOYEE_ABATE = 'Y' +) +SELECT AddressNumber, + UserID, + FullName, + DateUpdated, + TimeUpdated + FROM USER_CTE + WHERE RN = 1 diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDERS.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDERS.sql new file mode 100644 index 0000000..19247ad --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDERS.sql @@ -0,0 +1,23 @@ +-- WorkOrder Full Query +-- Source: JDESTAGE.F4801_VIEW +-- Destination: WorkOrder_Curr +-- Schedule: Mass/Daily/Hourly + +SELECT wo.DOCUMENTORDERINVOICEE_WADOCO AS WorkOrderNumber, + TRIM(wo.COSTCENTERALT_WAMMCU) AS BranchCode, + TRIM(wo.LOT_WALOTN) AS LotNumber, + TRIM(wo.IDENTIFIER2NDITEM_WALITM) AS ItemNumber, + wo.IDENTIFIERSHORTITEM_WAITM AS ShortItemNumber, + TRIM(wo.PARENTWONUMBER_WAPARS) AS ParentWorkOrderNumber, + wo.UNITSTRANSACTIONQTY_WAUORG AS OrderQuantity, + wo.UNITSQUANBACKORHELD_WASOBK AS HeldQuantity, + wo.UNITSQUANTITYCANCELED_WASOCN AS ScrappedQuantity, + wo.UNITSQUANTITYSHIPPED_WASOQS AS ShippedQuantity, + TRIM(wo.STATUSCODEWO_WASRST) AS StatusCode, + wo.DATESTATUSCHANGED_WADCG AS StatusCodeUpdateDT, + wo.DATETRANSACTIONJULIAN_WATRDJ AS IssueDate, + wo.DATESTART_WASTRT AS StartDate, + TRIM(wo.TYPEROUTING_WATRT) AS RoutingType, + wo.DATEUPDATED_WAUPMJ AS DateUpdated, + wo.TIMEOFDAY_WATDAY AS TimeUpdated + FROM JDESTAGE.F4801_VIEW wo diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDERS_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDERS_FILTERED.sql new file mode 100644 index 0000000..fa445da --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDERS_FILTERED.sql @@ -0,0 +1,28 @@ +-- WorkOrder Filtered Query (Incremental) +-- Source: JDESTAGE.F4801_VIEW +-- Destination: WorkOrder_Curr +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT wo.DOCUMENTORDERINVOICEE_WADOCO AS WorkOrderNumber, + TRIM(wo.COSTCENTERALT_WAMMCU) AS BranchCode, + TRIM(wo.LOT_WALOTN) AS LotNumber, + TRIM(wo.IDENTIFIER2NDITEM_WALITM) AS ItemNumber, + wo.IDENTIFIERSHORTITEM_WAITM AS ShortItemNumber, + TRIM(wo.PARENTWONUMBER_WAPARS) AS ParentWorkOrderNumber, + wo.UNITSTRANSACTIONQTY_WAUORG AS OrderQuantity, + wo.UNITSQUANBACKORHELD_WASOBK AS HeldQuantity, + wo.UNITSQUANTITYCANCELED_WASOCN AS ScrappedQuantity, + wo.UNITSQUANTITYSHIPPED_WASOQS AS ShippedQuantity, + TRIM(wo.STATUSCODEWO_WASRST) AS StatusCode, + wo.DATESTATUSCHANGED_WADCG AS StatusCodeUpdateDT, + wo.DATETRANSACTIONJULIAN_WATRDJ AS IssueDate, + wo.DATESTART_WASTRT AS StartDate, + TRIM(wo.TYPEROUTING_WATRT) AS RoutingType, + wo.DATEUPDATED_WAUPMJ AS DateUpdated, + wo.TIMEOFDAY_WATDAY AS TimeUpdated + FROM JDESTAGE.F4801_VIEW wo + WHERE ( + wo.DATEUPDATED_WAUPMJ > :dateUpdated OR + (wo.DATEUPDATED_WAUPMJ = :dateUpdated AND wo.TIMEOFDAY_WATDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDER_COMPONENTS.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDER_COMPONENTS.sql new file mode 100644 index 0000000..a1bc966 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDER_COMPONENTS.sql @@ -0,0 +1,15 @@ +-- WorkOrderComponent Full Query +-- Source: JDESTAGE.F3111_VIEW +-- Destination: WorkOrderComponent_Curr +-- Schedule: Mass/Daily/Hourly + +SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, + woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, + TRIM(woc.LOT_WMLOTN) AS LotNumber, + TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, + woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, + woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, + woc.DATEUPDATED_WMUPMJ AS DateUpdated, + woc.TIMEOFDAY_WMTDAY AS TimeUpdated + FROM JDESTAGE.F3111_VIEW woc + WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDER_COMPONENTS_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDER_COMPONENTS_FILTERED.sql new file mode 100644 index 0000000..02673df --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDER_COMPONENTS_FILTERED.sql @@ -0,0 +1,20 @@ +-- WorkOrderComponent Filtered Query (Incremental) +-- Source: JDESTAGE.F3111_VIEW +-- Destination: WorkOrderComponent_Curr +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT woc.UNIQUEKEYIDINTERNAL_WMUKID AS UniqueID, + woc.DOCUMENTORDERINVOICEE_WMDOCO AS WorkOrderNumber, + TRIM(woc.LOT_WMLOTN) AS LotNumber, + TRIM(woc.BRANCHCOMPONENT_WMCMCU) AS BranchCode, + woc.COMPONENTITEMNOSHORT_WMCPIT AS ShortItemNumber, + woc.QUANTITYTRANSACTION_WMTRQT AS Quantity, + woc.DATEUPDATED_WMUPMJ AS DateUpdated, + woc.TIMEOFDAY_WMTDAY AS TimeUpdated + FROM JDESTAGE.F3111_VIEW woc + WHERE TRIM(woc.LOT_WMLOTN) IS NOT NULL AND + ( + woc.DATEUPDATED_WMUPMJ > :dateUpdated OR + (woc.DATEUPDATED_WMUPMJ = :dateUpdated AND woc.TIMEOFDAY_WMTDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDER_ROUTING.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDER_ROUTING.sql new file mode 100644 index 0000000..fbb72c8 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDER_ROUTING.sql @@ -0,0 +1,24 @@ +-- WorkOrderRouting Full Query +-- Source: JDESTAGE.F3112Z1_VIEW +-- Destination: WorkOrderRouting +-- Schedule: Mass/Daily/Hourly + +SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, + TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, + TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, + woz.EDILINENUMBER_SZEDLN AS LineNumber, + woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, + TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, + woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, + TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, + TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, + TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, + TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, + woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, + woz.DATEUPDATED_SZUPMJ AS DateUpdated, + woz.TIMEOFDAY_SZTDAY AS TimeUpdated + FROM JDESTAGE.F3112Z1_VIEW woz + WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND + woz.DIRECTIONINDICATOR_SZDRIN = '2' AND + woz.TRANSACTIONACTION_SZTNAC = '02' AND + woz.PROGRAMID_SZPID = 'ER31410' diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDER_ROUTING_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDER_ROUTING_FILTERED.sql new file mode 100644 index 0000000..7b18548 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDER_ROUTING_FILTERED.sql @@ -0,0 +1,29 @@ +-- WorkOrderRouting Filtered Query (Incremental) +-- Source: JDESTAGE.F3112Z1_VIEW +-- Destination: WorkOrderRouting +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT TRIM(woz.EDIUSERID_SZEDUS) AS UserID, + TRIM(woz.EDIBATCHNUMBER_SZEDBT) AS BatchNumber, + TRIM(woz.EDITRANSACTNUMBER_SZEDTN) AS TransactionNumber, + woz.EDILINENUMBER_SZEDLN AS LineNumber, + woz.SEQUENCENOOPERATIONS_SZOPSQ AS StepNumber, + TRIM(woz.COSTCENTER_SZMCU) AS WorkCenterCode, + woz.DOCUMENTORDERINVOICEE_SZDOCO AS WorkOrderNumber, + TRIM(woz.TYPEROUTING_SZTRT) AS RoutingType, + TRIM(woz.COSTCENTERALT_SZMMCU) AS BranchCode, + TRIM(woz.DESCRIPTIONLINE1_SZDSC1) AS StepDescription, + TRIM(woz.USERRESERVEDREFERENCE_SZURRF) AS FunctionCode, + woz.DATETRANSACTIONJULIAN_SZTRDJ AS TransactionDate, + woz.DATEUPDATED_SZUPMJ AS DateUpdated, + woz.TIMEOFDAY_SZTDAY AS TimeUpdated + FROM JDESTAGE.F3112Z1_VIEW woz + WHERE woz.TYPETRANSACTION_SZTYTN = 'JDERTG' AND + woz.DIRECTIONINDICATOR_SZDRIN = '2' AND + woz.TRANSACTIONACTION_SZTNAC = '02' AND + woz.PROGRAMID_SZPID = 'ER31410' AND + ( + woz.DATEUPDATED_SZUPMJ > :dateUpdated OR + (woz.DATEUPDATED_SZUPMJ = :dateUpdated AND woz.TIMEOFDAY_SZTDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDER_STEP.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDER_STEP.sql new file mode 100644 index 0000000..503e08a --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDER_STEP.sql @@ -0,0 +1,19 @@ +-- WorkOrderStep Full Query +-- Source: JDESTAGE.F3112_VIEW, JDESTAGE.F00192_VIEW +-- Destination: WorkOrderStep_Curr +-- Schedule: Mass/Daily/Hourly + +SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, + TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, + TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, + wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, + TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, + TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, + wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, + CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, + CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, + TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, + wos.DATEUPDATED_WLUPMJ AS DateUpdated, + wos.TIMEOFDAY_WLTDAY AS TimeUpdated + FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN + JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY) diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDER_STEP_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDER_STEP_FILTERED.sql new file mode 100644 index 0000000..af788b8 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDER_STEP_FILTERED.sql @@ -0,0 +1,24 @@ +-- WorkOrderStep Filtered Query (Incremental) +-- Source: JDESTAGE.F3112_VIEW, JDESTAGE.F00192_VIEW +-- Destination: WorkOrderStep_Curr +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT wos.DOCUMENTORDERINVOICEE_WLDOCO AS WorkOrderNumber, + TRIM(wos.COSTCENTERALT_WLMMCU) AS BranchCode, + TRIM(wos.COSTCENTER_WLMCU) AS WorkCenterCode, + wos.SEQUENCENOOPERATIONS_WLOPSQ AS StepNumber, + TRIM(wos.DESCRIPTIONLINE1_WLDSC1) AS StepDescription, + TRIM(mes.DESCRIPT80CHARACTERS_CFDS80) AS FunctionOperationDescription, + wos.TYPEOPERATIONCODE_WLOPSC AS StepTypeCode, + CASE wos.DATESTART_WLSTRT WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATESTART_WLSTRT END AS StartDT, + CASE wos.DATECOMPLETION_WLSTRX WHEN TO_DATE('1900-01-01', 'yyyy-MM-dd') THEN NULL ELSE wos.DATECOMPLETION_WLSTRX END AS EndDT, + TRIM(wos.USERRESERVEDREFERENCE_WLURRF) AS FunctionCode, + wos.DATEUPDATED_WLUPMJ AS DateUpdated, + wos.TIMEOFDAY_WLTDAY AS TimeUpdated + FROM JDESTAGE.F3112_VIEW wos LEFT OUTER JOIN + JDESTAGE.F00192_VIEW mes ON (wos.USERRESERVEDREFERENCE_WLURRF = mes.USERDEFINEDCODE_CFKY) + WHERE ( + wos.DATEUPDATED_WLUPMJ > :dateUpdated OR + (wos.DATEUPDATED_WLUPMJ = :dateUpdated AND wos.TIMEOFDAY_WLTDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDER_TIMES.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDER_TIMES.sql new file mode 100644 index 0000000..2fb57b5 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDER_TIMES.sql @@ -0,0 +1,14 @@ +-- WorkOrderTime Full Query +-- Source: JDESTAGE.F31122_VIEW +-- Destination: WorkOrderTime_Curr +-- Schedule: Mass/Daily/Hourly + +SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, + TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, + wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, + wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, + wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, + wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, + wot.DATEUPDATED_WTUPMJ AS DateUpdated, + wot.TIMEOFDAY_WTTDAY AS TimeUpdated + FROM JDESTAGE.F31122_VIEW wot diff --git a/DATA_SYNC/JDE/SQL_GET_WORKORDER_TIMES_FILTERED.sql b/DATA_SYNC/JDE/SQL_GET_WORKORDER_TIMES_FILTERED.sql new file mode 100644 index 0000000..42f5660 --- /dev/null +++ b/DATA_SYNC/JDE/SQL_GET_WORKORDER_TIMES_FILTERED.sql @@ -0,0 +1,19 @@ +-- WorkOrderTime Filtered Query (Incremental) +-- Source: JDESTAGE.F31122_VIEW +-- Destination: WorkOrderTime_Curr +-- Schedule: Daily/Hourly (incremental merge) +-- Parameters: :dateUpdated (JDE date), :timeUpdated (JDE time) + +SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, + TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, + wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, + wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, + wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, + wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, + wot.DATEUPDATED_WTUPMJ AS DateUpdated, + wot.TIMEOFDAY_WTTDAY AS TimeUpdated + FROM JDESTAGE.F31122_VIEW wot + WHERE ( + wot.DATEUPDATED_WTUPMJ > :dateUpdated OR + (wot.DATEUPDATED_WTUPMJ = :dateUpdated AND wot.TIMEOFDAY_WTTDAY >= :timeUpdated) + ) diff --git a/DATA_SYNC/JDE_ARCHIVE/SQL_GET_LOT_USAGES.sql b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_LOT_USAGES.sql new file mode 100644 index 0000000..2361dc7 --- /dev/null +++ b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_LOT_USAGES.sql @@ -0,0 +1,30 @@ +-- LotUsage Archive Query +-- Source: QADTA.F4111 (current) + ARCDTAQA.F4111 (archived) +-- Used by: GetLotUsagesArchive() - on-demand historical retrieval +-- Note: Not scheduled, used for historical lookups + +SELECT lu.ILUKID AS UniqueID, + lu.ILDOCO AS WorkOrderNumber, + TRIM(lu.ILLOTN) AS LotNumber, + TRIM(lu.ILMCU) AS BranchCode, + lu.ILITM AS ShortItemNumber, + lu.ILTRQT AS Quantity, + lu.ILTRDJ AS DateUpdated, + lu.ILTDAY AS TimeUpdated + FROM QADTA.F4111 lu + WHERE lu.ILDCT = 'IM' AND + TRIM(lu.ILLOTN) IS NOT NULL + +UNION ALL + +SELECT lu.ILUKID AS UniqueID, + lu.ILDOCO AS WorkOrderNumber, + TRIM(lu.ILLOTN) AS LotNumber, + TRIM(lu.ILMCU) AS BranchCode, + lu.ILITM AS ShortItemNumber, + lu.ILTRQT AS Quantity, + lu.ILTRDJ AS DateUpdated, + lu.ILTDAY AS TimeUpdated + FROM ARCDTAQA.F4111 lu + WHERE lu.ILDCT = 'IM' AND + TRIM(lu.ILLOTN) IS NOT NULL diff --git a/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDERS.sql b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDERS.sql new file mode 100644 index 0000000..735be16 --- /dev/null +++ b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDERS.sql @@ -0,0 +1,44 @@ +-- WorkOrder Archive Query +-- Source: QADTA.F4801 (current) + ARCDTAQA.F4801 (archived) +-- Used by: GetWorkOrdersArchive() - on-demand historical retrieval +-- Note: Not scheduled, used for historical lookups + +SELECT wo.WADOCO AS WorkOrderNumber, + TRIM(wo.WAMMCU) AS BranchCode, + TRIM(wo.WALOTN) AS LotNumber, + TRIM(wo.WALITM) AS ItemNumber, + wo.WAITM AS ShortItemNumber, + TRIM(wo.WAPARS) AS ParentWorkOrderNumber, + wo.WAUORG AS OrderQuantity, + wo.WASOBK AS HeldQuantity, + wo.WASOCN AS ScrappedQuantity, + wo.WASOQS AS ShippedQuantity, + TRIM(wo.WASRST) AS StatusCode, + wo.WADCG AS StatusCodeUpdateDT, + wo.WATRDJ AS IssueDate, + wo.WASTRT AS StartDate, + TRIM(wo.WATRT) AS RoutingType, + wo.WAUPMJ AS DateUpdated, + wo.WATDAY AS TimeUpdated + FROM QADTA.F4801 wo + +UNION ALL + +SELECT wo.WADOCO AS WorkOrderNumber, + TRIM(wo.WAMMCU) AS BranchCode, + TRIM(wo.WALOTN) AS LotNumber, + TRIM(wo.WALITM) AS ItemNumber, + wo.WAITM AS ShortItemNumber, + TRIM(wo.WAPARS) AS ParentWorkOrderNumber, + wo.WAUORG AS OrderQuantity, + wo.WASOBK AS HeldQuantity, + wo.WASOCN AS ScrappedQuantity, + wo.WASOQS AS ShippedQuantity, + TRIM(wo.WASRST) AS StatusCode, + wo.WADCG AS StatusCodeUpdateDT, + wo.WATRDJ AS IssueDate, + wo.WASTRT AS StartDate, + TRIM(wo.WATRT) AS RoutingType, + wo.WAUPMJ AS DateUpdated, + wo.WATDAY AS TimeUpdated + FROM ARCDTAQA.F4801 wo diff --git a/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDER_COMPONENTS.sql b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDER_COMPONENTS.sql new file mode 100644 index 0000000..4d29b0d --- /dev/null +++ b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDER_COMPONENTS.sql @@ -0,0 +1,28 @@ +-- WorkOrderComponent Archive Query +-- Source: QADTA.F3111 (current) + ARCDTAQA.F3111 (archived) +-- Used by: GetWorkOrderComponentsArchive() - on-demand historical retrieval +-- Note: Not scheduled, used for historical lookups + +SELECT woc.WMUKID AS UniqueID, + woc.WMDOCO AS WorkOrderNumber, + TRIM(woc.WMLOTN) AS LotNumber, + TRIM(woc.WMCMCU) AS BranchCode, + woc.WMCPIT AS ShortItemNumber, + woc.WMTRQT AS Quantity, + woc.WMUPMJ AS DateUpdated, + woc.WMTDAY AS TimeUpdated + FROM QADTA.F3111 woc + WHERE TRIM(woc.WMLOTN) IS NOT NULL + +UNION ALL + +SELECT woc.WMUKID AS UniqueID, + woc.WMDOCO AS WorkOrderNumber, + TRIM(woc.WMLOTN) AS LotNumber, + TRIM(woc.WMCMCU) AS BranchCode, + woc.WMCPIT AS ShortItemNumber, + woc.WMTRQT AS Quantity, + woc.WMUPMJ AS DateUpdated, + woc.WMTDAY AS TimeUpdated + FROM ARCDTAQA.F3111 woc + WHERE TRIM(woc.WMLOTN) IS NOT NULL diff --git a/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDER_STEP.sql b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDER_STEP.sql new file mode 100644 index 0000000..2aab768 --- /dev/null +++ b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDER_STEP.sql @@ -0,0 +1,37 @@ +-- WorkOrderStep Archive Query +-- Source: QADTA.F3112 + QADTA.F00192 (current) + ARCDTAQA.F3112 (archived) +-- Used by: GetWorkOrderStepsArchive() - on-demand historical retrieval +-- Note: Not scheduled, used for historical lookups +-- Note: Function code lookup uses QADTA.F00192 for both current and archived data + +SELECT wos.WLDOCO AS WorkOrderNumber, + TRIM(wos.WLMMCU) AS BranchCode, + TRIM(wos.WLMCU) AS WorkCenterCode, + wos.WLOPSQ AS StepNumber, + TRIM(wos.WLDSC1) AS StepDescription, + TRIM(mes.CFDS80) AS FunctionOperationDescription, + wos.WLOPSC AS StepTypeCode, + CASE wos.WLSTRT WHEN 0 THEN NULL ELSE wos.WLSTRT END AS StartDT, + CASE wos.WLSTRX WHEN 0 THEN NULL ELSE wos.WLSTRX END AS EndDT, + TRIM(wos.WLURRF) AS FunctionCode, + wos.WLUPMJ AS DateUpdated, + wos.WLTDAY AS TimeUpdated + FROM QADTA.F3112 wos LEFT OUTER JOIN + QADTA.F00192 mes ON (wos.WLURRF = mes.CFKY) + +UNION ALL + +SELECT wos.WLDOCO AS WorkOrderNumber, + TRIM(wos.WLMMCU) AS BranchCode, + TRIM(wos.WLMCU) AS WorkCenterCode, + wos.WLOPSQ AS StepNumber, + TRIM(wos.WLDSC1) AS StepDescription, + TRIM(mes.CFDS80) AS FunctionOperationDescription, + wos.WLOPSC AS StepTypeCode, + CASE wos.WLSTRT WHEN 0 THEN NULL ELSE wos.WLSTRT END AS StartDT, + CASE wos.WLSTRX WHEN 0 THEN NULL ELSE wos.WLSTRX END AS EndDT, + TRIM(wos.WLURRF) AS FunctionCode, + wos.WLUPMJ AS DateUpdated, + wos.WLTDAY AS TimeUpdated + FROM ARCDTAQA.F3112 wos LEFT OUTER JOIN + QADTA.F00192 mes ON (wos.WLURRF = mes.CFKY) diff --git a/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDER_TIMES.sql b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDER_TIMES.sql new file mode 100644 index 0000000..2ae66c7 --- /dev/null +++ b/DATA_SYNC/JDE_ARCHIVE/SQL_GET_WORKORDER_TIMES.sql @@ -0,0 +1,26 @@ +-- WorkOrderTime Archive Query +-- Source: QADTA.F31122 (current) + ARCDTAQA.F31122 (archived) +-- Used by: GetWorkOrderTimesArchive() - on-demand historical retrieval +-- Note: Not scheduled, used for historical lookups + +SELECT wot.WTUKID AS UniqueID, + TRIM(wot.WTMMCU) AS BranchCode, + wot.WTDOCO AS WorkOrderNumber, + wot.WTOPSQ AS StepNumber, + wot.WTAN8 AS AddressNumber, + wot.WTDGL AS GlDate, + wot.WTUPMJ AS DateUpdated, + wot.WTTDAY AS TimeUpdated + FROM QADTA.F31122 wot + +UNION ALL + +SELECT wot.WTUKID AS UniqueID, + TRIM(wot.WTMMCU) AS BranchCode, + wot.WTDOCO AS WorkOrderNumber, + wot.WTOPSQ AS StepNumber, + wot.WTAN8 AS AddressNumber, + wot.WTDGL AS GlDate, + wot.WTUPMJ AS DateUpdated, + wot.WTTDAY AS TimeUpdated + FROM ARCDTAQA.F31122 wot diff --git a/NEW/src/JdeScoping.Api/Controllers/AuthController.cs b/NEW/src/JdeScoping.Api/Controllers/AuthController.cs index 2a62491..6b3180a 100644 --- a/NEW/src/JdeScoping.Api/Controllers/AuthController.cs +++ b/NEW/src/JdeScoping.Api/Controllers/AuthController.cs @@ -1,6 +1,7 @@ using System.Security.Claims; using System.Text.Json; using JdeScoping.Api.Extensions; +using JdeScoping.Core.ApiContracts; using JdeScoping.Core.Interfaces; using JdeScoping.Core.Models; using JdeScoping.Core.Models.Auth; @@ -16,7 +17,7 @@ namespace JdeScoping.Api.Controllers; /// /// Authentication endpoints for Blazor WASM client /// -[Route("api/auth")] +[Route(ApiRoutes.Auth.Base)] [ApiController] public class AuthController : ApiControllerBase { diff --git a/NEW/src/JdeScoping.Api/Controllers/FileController.cs b/NEW/src/JdeScoping.Api/Controllers/FileController.cs index 069c798..3fd3cbf 100644 --- a/NEW/src/JdeScoping.Api/Controllers/FileController.cs +++ b/NEW/src/JdeScoping.Api/Controllers/FileController.cs @@ -1,3 +1,4 @@ +using JdeScoping.Core.ApiContracts; using JdeScoping.Core.Interfaces; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; @@ -10,7 +11,7 @@ namespace JdeScoping.Api.Controllers; /// [Authorize] [ApiController] -[Route("api/fileio")] +[Route(ApiRoutes.FileIO.Base)] public partial class FileIOController : ApiControllerBase { private const string ContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; diff --git a/NEW/src/JdeScoping.Api/Controllers/LookupController.cs b/NEW/src/JdeScoping.Api/Controllers/LookupController.cs index 24e7252..e33a9f3 100644 --- a/NEW/src/JdeScoping.Api/Controllers/LookupController.cs +++ b/NEW/src/JdeScoping.Api/Controllers/LookupController.cs @@ -1,3 +1,4 @@ +using JdeScoping.Core.ApiContracts; using JdeScoping.Core.Interfaces; using JdeScoping.Core.ViewModels; using Microsoft.AspNetCore.Http; @@ -8,7 +9,7 @@ namespace JdeScoping.Api.Controllers; /// /// Lookup/autocomplete endpoints (no authorization required) /// -[Route("api/lookup")] +[Route(ApiRoutes.Lookup.Base)] [ApiController] public class LookupController : ApiControllerBase { diff --git a/NEW/src/JdeScoping.Api/Controllers/SearchController.cs b/NEW/src/JdeScoping.Api/Controllers/SearchController.cs index a077f27..7b71742 100644 --- a/NEW/src/JdeScoping.Api/Controllers/SearchController.cs +++ b/NEW/src/JdeScoping.Api/Controllers/SearchController.cs @@ -1,4 +1,5 @@ using JdeScoping.Api.Hubs; +using JdeScoping.Core.ApiContracts; using JdeScoping.Core.Interfaces; using JdeScoping.Core.Models; using JdeScoping.Core.Models.Enums; @@ -15,7 +16,7 @@ namespace JdeScoping.Api.Controllers; /// /// Search management controller /// -[Route("api/search")] +[Route(ApiRoutes.Search.Base)] [ApiController] [Authorize] public class SearchController : ApiControllerBase diff --git a/NEW/src/JdeScoping.Client/Services/AuthApiClient.cs b/NEW/src/JdeScoping.Client/Services/AuthApiClient.cs new file mode 100644 index 0000000..03407e3 --- /dev/null +++ b/NEW/src/JdeScoping.Client/Services/AuthApiClient.cs @@ -0,0 +1,26 @@ +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.Models; +using JdeScoping.Core.Models.Auth; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of IAuthApiClient. +/// +public class AuthApiClient : ApiClientBase, IAuthApiClient +{ + public AuthApiClient(HttpClient httpClient) : base(httpClient) { } + + public Task> GetPublicKeyAsync(CancellationToken ct = default) + => GetAsync(ApiRoutes.Auth.PublicKey, ct); + + public Task> LoginAsync(EncryptedLoginRequest request, CancellationToken ct = default) + => PostAsync(ApiRoutes.Auth.Login, request, ct); + + public Task> LogoutAsync(CancellationToken ct = default) + => PostAsync(ApiRoutes.Auth.Logout, ct); + + public Task> GetCurrentUserAsync(CancellationToken ct = default) + => GetAsync(ApiRoutes.Auth.Me, ct); +} diff --git a/NEW/src/JdeScoping.Client/Services/FileApiClient.cs b/NEW/src/JdeScoping.Client/Services/FileApiClient.cs new file mode 100644 index 0000000..58c371e --- /dev/null +++ b/NEW/src/JdeScoping.Client/Services/FileApiClient.cs @@ -0,0 +1,41 @@ +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of IFileApiClient. +/// +public class FileApiClient : ApiClientBase, IFileApiClient +{ + public FileApiClient(HttpClient httpClient) : base(httpClient) { } + + // Downloads + + public Task> DownloadWorkOrdersTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default) + => PostForBytesAsync(ApiRoutes.FileIO.DownloadWorkOrders, existingData, ct); + + public Task> DownloadItemsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default) + => PostForBytesAsync(ApiRoutes.FileIO.DownloadItems, existingData, ct); + + public Task> DownloadComponentLotsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default) + => PostForBytesAsync(ApiRoutes.FileIO.DownloadComponentLots, existingData, ct); + + public Task> DownloadPartOperationsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default) + => PostForBytesAsync(ApiRoutes.FileIO.DownloadPartOperations, existingData, ct); + + // Uploads + + public Task>> UploadWorkOrdersAsync(Stream fileStream, string fileName, CancellationToken ct = default) + => PostMultipartAsync>(ApiRoutes.FileIO.UploadWorkOrders, fileStream, fileName, ct); + + public Task>> UploadItemsAsync(Stream fileStream, string fileName, CancellationToken ct = default) + => PostMultipartAsync>(ApiRoutes.FileIO.UploadItems, fileStream, fileName, ct); + + public Task>> UploadComponentLotsAsync(Stream fileStream, string fileName, CancellationToken ct = default) + => PostMultipartAsync>(ApiRoutes.FileIO.UploadComponentLots, fileStream, fileName, ct); + + public Task>> UploadPartOperationsAsync(Stream fileStream, string fileName, CancellationToken ct = default) + => PostMultipartAsync>(ApiRoutes.FileIO.UploadPartOperations, fileStream, fileName, ct); +} diff --git a/NEW/src/JdeScoping.Client/Services/LookupApiClient.cs b/NEW/src/JdeScoping.Client/Services/LookupApiClient.cs new file mode 100644 index 0000000..ec50a54 --- /dev/null +++ b/NEW/src/JdeScoping.Client/Services/LookupApiClient.cs @@ -0,0 +1,25 @@ +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of ILookupApiClient. +/// +public class LookupApiClient : ApiClientBase, ILookupApiClient +{ + public LookupApiClient(HttpClient httpClient) : base(httpClient) { } + + public Task>> FindItemsAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindItems(query), ct); + + public Task>> FindProfitCentersAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindProfitCenters(query), ct); + + public Task>> FindWorkCentersAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindWorkCenters(query), ct); + + public Task>> FindOperatorsAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindOperators(query), ct); +} diff --git a/NEW/src/JdeScoping.Client/Services/SearchApiClient.cs b/NEW/src/JdeScoping.Client/Services/SearchApiClient.cs new file mode 100644 index 0000000..258d589 --- /dev/null +++ b/NEW/src/JdeScoping.Client/Services/SearchApiClient.cs @@ -0,0 +1,31 @@ +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of ISearchApiClient. +/// +public class SearchApiClient : ApiClientBase, ISearchApiClient +{ + public SearchApiClient(HttpClient httpClient) : base(httpClient) { } + + public Task>> GetUserSearchesAsync(CancellationToken ct = default) + => GetAsync>(ApiRoutes.Search.Base, ct); + + public Task>> GetQueuedSearchesAsync(CancellationToken ct = default) + => GetAsync>(ApiRoutes.Search.Queue, ct); + + public Task> GetSearchAsync(int id, CancellationToken ct = default) + => GetAsync(ApiRoutes.Search.GetById(id), ct); + + public Task> CopySearchAsync(int id, CancellationToken ct = default) + => GetAsync(ApiRoutes.Search.GetCopy(id), ct); + + public Task> CreateSearchAsync(SearchViewModel search, CancellationToken ct = default) + => PostAsync(ApiRoutes.Search.Base, search, ct); + + public Task> GetResultsAsync(int id, CancellationToken ct = default) + => GetBytesAsync(ApiRoutes.Search.GetResults(id), ct); +} diff --git a/NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs b/NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs new file mode 100644 index 0000000..99b88f5 --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs @@ -0,0 +1,124 @@ +namespace JdeScoping.Core.ApiContracts; + +/// +/// Shared API route constants. Use in controller attributes and client implementations. +/// +public static class ApiRoutes +{ + /// + /// Routes for search API endpoints. + /// + public static class Search + { + /// Base route for search endpoints. + public const string Base = "api/search"; + + /// Route for queued searches. + public const string Queue = "api/search/queue"; + + /// Route template for getting a search by ID (use in controller attributes). + public const string ById = "{id:int}"; + + /// Route template for copying a search (use in controller attributes). + public const string Copy = "{id:int}/copy"; + + /// Route template for getting search results (use in controller attributes). + public const string Results = "{id:int}/results"; + + /// Builds the route to get a specific search. + public static string GetById(int id) => $"api/search/{id}"; + + /// Builds the route to copy a search. + public static string GetCopy(int id) => $"api/search/{id}/copy"; + + /// Builds the route to get search results. + public static string GetResults(int id) => $"api/search/{id}/results"; + } + + /// + /// Routes for lookup/autocomplete API endpoints. + /// + public static class Lookup + { + /// Base route for lookup endpoints. + public const string Base = "api/lookup"; + + /// Route for item lookup. + public const string Items = "api/lookup/items"; + + /// Route for profit center lookup. + public const string ProfitCenters = "api/lookup/profit-centers"; + + /// Route for work center lookup. + public const string WorkCenters = "api/lookup/work-centers"; + + /// Route for operator lookup. + public const string Operators = "api/lookup/operators"; + + /// Builds the route to find items with URL-encoded query. + public static string FindItems(string query) => $"{Items}?q={Uri.EscapeDataString(query)}"; + + /// Builds the route to find profit centers with URL-encoded query. + public static string FindProfitCenters(string query) => $"{ProfitCenters}?q={Uri.EscapeDataString(query)}"; + + /// Builds the route to find work centers with URL-encoded query. + public static string FindWorkCenters(string query) => $"{WorkCenters}?q={Uri.EscapeDataString(query)}"; + + /// Builds the route to find operators with URL-encoded query. + public static string FindOperators(string query) => $"{Operators}?q={Uri.EscapeDataString(query)}"; + } + + /// + /// Routes for authentication API endpoints. + /// + public static class Auth + { + /// Base route for auth endpoints. + public const string Base = "api/auth"; + + /// Route to get the public key for credential encryption. + public const string PublicKey = "api/auth/public-key"; + + /// Route for login. + public const string Login = "api/auth/login"; + + /// Route for logout. + public const string Logout = "api/auth/logout"; + + /// Route to get current user info. + public const string Me = "api/auth/me"; + } + + /// + /// Routes for file upload/download API endpoints. + /// + public static class FileIO + { + /// Base route for file IO endpoints. + public const string Base = "api/fileio"; + + /// Route to download work orders template. + public const string DownloadWorkOrders = "api/fileio/workorders/download"; + + /// Route to download items template. + public const string DownloadItems = "api/fileio/items/download"; + + /// Route to download component lots template. + public const string DownloadComponentLots = "api/fileio/componentlots/download"; + + /// Route to download part operations template. + public const string DownloadPartOperations = "api/fileio/partoperations/download"; + + /// Route to upload work orders. + public const string UploadWorkOrders = "api/fileio/workorders/upload"; + + /// Route to upload items. + public const string UploadItems = "api/fileio/items/upload"; + + /// Route to upload component lots. + public const string UploadComponentLots = "api/fileio/componentlots/upload"; + + /// Route to upload part operations. + public const string UploadPartOperations = "api/fileio/partoperations/upload"; + } +} diff --git a/NEW/src/JdeScoping.Core/ApiContracts/IAuthApiClient.cs b/NEW/src/JdeScoping.Core/ApiContracts/IAuthApiClient.cs new file mode 100644 index 0000000..1e5221f --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/IAuthApiClient.cs @@ -0,0 +1,23 @@ +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.Models; +using JdeScoping.Core.Models.Auth; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for authentication API operations. +/// +public interface IAuthApiClient +{ + /// Gets the server's RSA public key for encrypting login credentials. + Task> GetPublicKeyAsync(CancellationToken ct = default); + + /// Authenticates with encrypted credentials. + Task> LoginAsync(EncryptedLoginRequest request, CancellationToken ct = default); + + /// Logs out the current user. + Task> LogoutAsync(CancellationToken ct = default); + + /// Gets the current authenticated user's information. + Task> GetCurrentUserAsync(CancellationToken ct = default); +} diff --git a/NEW/src/JdeScoping.Core/ApiContracts/IFileApiClient.cs b/NEW/src/JdeScoping.Core/ApiContracts/IFileApiClient.cs new file mode 100644 index 0000000..e81b18d --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/IFileApiClient.cs @@ -0,0 +1,39 @@ +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for file upload/download API operations. +/// Note: Uses Stream for client-side; controllers use IFormFile. +/// +public interface IFileApiClient +{ + // Downloads (POST with existing data, returns Excel bytes) + + /// Downloads work orders template, optionally pre-filled with existing data. + Task> DownloadWorkOrdersTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + + /// Downloads items template, optionally pre-filled with existing data. + Task> DownloadItemsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + + /// Downloads component lots template, optionally pre-filled with existing data. + Task> DownloadComponentLotsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + + /// Downloads part operations template, optionally pre-filled with existing data. + Task> DownloadPartOperationsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + + // Uploads (multipart form, returns parsed data) + + /// Uploads work orders Excel file and returns parsed data. + Task>> UploadWorkOrdersAsync(Stream fileStream, string fileName, CancellationToken ct = default); + + /// Uploads items Excel file and returns parsed data. + Task>> UploadItemsAsync(Stream fileStream, string fileName, CancellationToken ct = default); + + /// Uploads component lots Excel file and returns parsed data. + Task>> UploadComponentLotsAsync(Stream fileStream, string fileName, CancellationToken ct = default); + + /// Uploads part operations Excel file and returns parsed data. + Task>> UploadPartOperationsAsync(Stream fileStream, string fileName, CancellationToken ct = default); +} diff --git a/NEW/src/JdeScoping.Core/ApiContracts/ILookupApiClient.cs b/NEW/src/JdeScoping.Core/ApiContracts/ILookupApiClient.cs new file mode 100644 index 0000000..6abd55e --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/ILookupApiClient.cs @@ -0,0 +1,22 @@ +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for lookup/autocomplete API operations. +/// +public interface ILookupApiClient +{ + /// Finds items matching the search query. + Task>> FindItemsAsync(string query, CancellationToken ct = default); + + /// Finds profit centers matching the search query. + Task>> FindProfitCentersAsync(string query, CancellationToken ct = default); + + /// Finds work centers matching the search query. + Task>> FindWorkCentersAsync(string query, CancellationToken ct = default); + + /// Finds operators (JDE users) matching the search query. + Task>> FindOperatorsAsync(string query, CancellationToken ct = default); +} diff --git a/NEW/src/JdeScoping.Core/ApiContracts/ISearchApiClient.cs b/NEW/src/JdeScoping.Core/ApiContracts/ISearchApiClient.cs new file mode 100644 index 0000000..2631fce --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/ISearchApiClient.cs @@ -0,0 +1,28 @@ +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for search API operations. +/// +public interface ISearchApiClient +{ + /// Gets all searches for the current user. + Task>> GetUserSearchesAsync(CancellationToken ct = default); + + /// Gets all queued searches. + Task>> GetQueuedSearchesAsync(CancellationToken ct = default); + + /// Gets a specific search by ID. + Task> GetSearchAsync(int id, CancellationToken ct = default); + + /// Copies an existing search to create a new one (returns copy without persisting). + Task> CopySearchAsync(int id, CancellationToken ct = default); + + /// Creates and submits a new search. + Task> CreateSearchAsync(SearchViewModel search, CancellationToken ct = default); + + /// Downloads the results for a completed search as Excel bytes. + Task> GetResultsAsync(int id, CancellationToken ct = default); +} diff --git a/NEW/src/JdeScoping.Core/ApiContracts/Results/ApiError.cs b/NEW/src/JdeScoping.Core/ApiContracts/Results/ApiError.cs new file mode 100644 index 0000000..001aefc --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/Results/ApiError.cs @@ -0,0 +1,8 @@ +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// General API error with message and optional status code. +/// +/// Error message describing what went wrong. +/// Optional HTTP status code. +public readonly record struct ApiError(string Message, int? StatusCode = null); diff --git a/NEW/src/JdeScoping.Core/ApiContracts/Results/ApiResult.cs b/NEW/src/JdeScoping.Core/ApiContracts/Results/ApiResult.cs new file mode 100644 index 0000000..090cf8d --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/Results/ApiResult.cs @@ -0,0 +1,39 @@ +using OneOf; + +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Standard API result type for client-side operations. +/// Represents either success with value T, or one of several error types. +/// +/// The success value type. +[GenerateOneOf] +public partial class ApiResult : OneOfBase +{ + /// Returns true if the result is a success value. + public bool IsSuccess => IsT0; + + /// Returns true if the result is NotFound. + public bool IsNotFound => IsT1; + + /// Returns true if the result is a ValidationError. + public bool IsValidationError => IsT2; + + /// Returns true if the result is Unauthorized. + public bool IsUnauthorized => IsT3; + + /// Returns true if the result is Forbidden. + public bool IsForbidden => IsT4; + + /// Returns true if the result is an ApiError. + public bool IsError => IsT5; + + /// Gets the success value. Throws if not a success. + public new T Value => AsT0; + + /// Gets the validation error. Throws if not a validation error. + public ValidationError ValidationError => AsT2; + + /// Gets the API error. Throws if not an API error. + public ApiError Error => AsT5; +} diff --git a/NEW/src/JdeScoping.Core/ApiContracts/Results/Forbidden.cs b/NEW/src/JdeScoping.Core/ApiContracts/Results/Forbidden.cs new file mode 100644 index 0000000..3d3ef81 --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/Results/Forbidden.cs @@ -0,0 +1,6 @@ +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Access denied (HTTP 403). +/// +public readonly record struct Forbidden; diff --git a/NEW/src/JdeScoping.Core/ApiContracts/Results/NotFound.cs b/NEW/src/JdeScoping.Core/ApiContracts/Results/NotFound.cs new file mode 100644 index 0000000..b9fb701 --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/Results/NotFound.cs @@ -0,0 +1,6 @@ +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Resource not found (HTTP 404). +/// +public readonly record struct NotFound; diff --git a/NEW/src/JdeScoping.Core/ApiContracts/Results/Unauthorized.cs b/NEW/src/JdeScoping.Core/ApiContracts/Results/Unauthorized.cs new file mode 100644 index 0000000..f550295 --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/Results/Unauthorized.cs @@ -0,0 +1,6 @@ +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Authentication required (HTTP 401). +/// +public readonly record struct Unauthorized; diff --git a/NEW/src/JdeScoping.Core/ApiContracts/Results/Unit.cs b/NEW/src/JdeScoping.Core/ApiContracts/Results/Unit.cs new file mode 100644 index 0000000..899acd7 --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/Results/Unit.cs @@ -0,0 +1,6 @@ +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Empty success type for void operations. +/// +public readonly record struct Unit; diff --git a/NEW/src/JdeScoping.Core/ApiContracts/Results/ValidationError.cs b/NEW/src/JdeScoping.Core/ApiContracts/Results/ValidationError.cs new file mode 100644 index 0000000..ddbe43a --- /dev/null +++ b/NEW/src/JdeScoping.Core/ApiContracts/Results/ValidationError.cs @@ -0,0 +1,15 @@ +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Validation failed (HTTP 400) with field-level errors. +/// Maps to ASP.NET Core ValidationProblemDetails format. +/// +/// Dictionary mapping field names to error messages. +public readonly record struct ValidationError(IReadOnlyDictionary FieldErrors) +{ + /// + /// Creates a ValidationError from a dictionary of field errors. + /// + public static ValidationError FromDictionary(Dictionary errors) + => new(errors); +} diff --git a/NEW/src/JdeScoping.Core/JdeScoping.Core.csproj b/NEW/src/JdeScoping.Core/JdeScoping.Core.csproj index 1f0f613..ff01dbd 100644 --- a/NEW/src/JdeScoping.Core/JdeScoping.Core.csproj +++ b/NEW/src/JdeScoping.Core/JdeScoping.Core.csproj @@ -8,6 +8,8 @@ + + diff --git a/NEW/src/JdeScoping.DataAccess/QueryBuilders/SqlKataSearchQueryBuilder.cs b/NEW/src/JdeScoping.DataAccess/QueryBuilders/SqlKataSearchQueryBuilder.cs index 34969a3..3cc545e 100644 --- a/NEW/src/JdeScoping.DataAccess/QueryBuilders/SqlKataSearchQueryBuilder.cs +++ b/NEW/src/JdeScoping.DataAccess/QueryBuilders/SqlKataSearchQueryBuilder.cs @@ -182,9 +182,9 @@ public sealed class SqlKataSearchQueryBuilder : ISearchQueryBuilder --Insert from work centers directly INSERT INTO #P_WorkCenters (Code) - SELECT Code - FROM dbo.fn_GetSearchWorkCenters(@SearchId) - WHERE NOT EXISTS (SELECT 1 FROM #P_WorkCenters WHERE Code = Code); + SELECT wc.Code + FROM dbo.fn_GetSearchWorkCenters(@SearchId) wc + WHERE NOT EXISTS (SELECT 1 FROM #P_WorkCenters pwc WHERE pwc.Code = wc.Code); """; } diff --git a/NEW/src/JdeScoping.DataSync/Etl/Destinations/DbBulkImportDestination.cs b/NEW/src/JdeScoping.DataSync/Etl/Destinations/DbBulkImportDestination.cs index 311b7ef..8ec9424 100644 --- a/NEW/src/JdeScoping.DataSync/Etl/Destinations/DbBulkImportDestination.cs +++ b/NEW/src/JdeScoping.DataSync/Etl/Destinations/DbBulkImportDestination.cs @@ -15,9 +15,12 @@ namespace JdeScoping.DataSync.Etl.Destinations; /// public class DbBulkImportDestination : IImportDestination { - private const int DefaultBatchSize = 10000; + private const int DefaultBatchSize = 100000; private const int DefaultCommandTimeoutSeconds = 600; + /// Use this for very large tables to avoid timeout during bulk copy. + public const int InfiniteTimeout = 0; + private readonly IDbConnectionFactory _connectionFactory; private readonly string _tableName; private readonly int _batchSize; @@ -73,8 +76,8 @@ public class DbBulkImportDestination : IImportDestination // Get destination columns for column mapping var destColumns = await GetDestinationColumnsAsync(connection, cancellationToken); - // Bulk copy data - using var bulkCopy = new SqlBulkCopy(connection) + // Bulk copy data with TableLock for reduced logging overhead + using var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.TableLock, null) { DestinationTableName = qualifiedName, BatchSize = _batchSize, @@ -98,8 +101,8 @@ public class DbBulkImportDestination : IImportDestination $"No columns from source exist in destination table '{_tableName}'. " + "Check column names match between source query and destination table."); - // Track rows via event - bulkCopy.NotifyAfter = _batchSize; + // Track rows via event (notify less frequently to reduce overhead) + bulkCopy.NotifyAfter = _batchSize * 10; bulkCopy.SqlRowsCopied += (_, e) => { totalRows = e.RowsCopied; diff --git a/PLANS/2026-01-03-dev-etl-pipeline-design.md b/PLANS/2026-01-03-dev-etl-pipeline-design.md new file mode 100644 index 0000000..ad006ce --- /dev/null +++ b/PLANS/2026-01-03-dev-etl-pipeline-design.md @@ -0,0 +1,236 @@ +# Development ETL Pipeline Design + +## Purpose + +Create development ETL pipelines that load data from cached `.json.zstd` files into the local SQL Server database. This enables local development and testing without requiring access to live Oracle/Sybase enterprise sources. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ JsonZstdFileSource │ +├─────────────────────────────────────────────────────────────┤ +│ File Path (.json.zstd) │ +│ │ │ +│ ▼ │ +│ ZstdSharp DecompressionStream │ +│ │ │ +│ ▼ │ +│ JsonStreamingDataReader : IDataReader │ +│ │ │ +│ ▼ │ +│ ETL Pipeline (transformers → destination) │ +└─────────────────────────────────────────────────────────────┘ +``` + +**Execution Flow:** +1. `JsonZstdFileSource` opens the `.json.zstd` file +2. `ZstdSharp.DecompressionStream` decompresses on-the-fly +3. `JsonStreamingDataReader` parses JSON array, yielding one row at a time +4. ETL pipeline applies transformers and writes to SQL Server via bulk copy + +## Components + +### JsonColumnSchema + +Column metadata record used by the streaming reader: + +```csharp +public record JsonColumnSchema( + string Name, + Type ClrType, + bool IsNullable = true); +``` + +### JsonStreamingDataReader + +Implements `IDataReader` to stream JSON array without loading into memory: + +```csharp +internal class JsonStreamingDataReader : IDataReader +{ + private readonly StreamReader _reader; + private readonly JsonColumnSchema[] _schema; + private readonly Dictionary _nameToOrdinal; + private object?[] _currentRow; + + public int FieldCount => _schema.Length; + public string GetName(int ordinal) => _schema[ordinal].Name; + public Type GetFieldType(int ordinal) => _schema[ordinal].ClrType; + public object GetValue(int ordinal) => _currentRow[ordinal] ?? DBNull.Value; + + public bool Read() + { + // Parse next JSON object from array + // Map properties to _currentRow by ordinal + // Return false at end of array + } +} +``` + +**Key Design Decisions:** +- Uses `JsonDocument.ParseValue()` to read one object at a time (memory efficient) +- Properties mapped to schema by name (case-insensitive) +- Missing properties become `DBNull.Value` +- Extra JSON properties are ignored + +### JsonZstdFileSource + +Implements `IImportSource` for the ETL pipeline: + +```csharp +public class JsonZstdFileSource : IImportSource +{ + private readonly string _filePath; + private readonly JsonColumnSchema[] _schema; + private FileStream? _fileStream; + private DecompressionStream? _decompressionStream; + + public string SourceName => $"JsonZstd:{Path.GetFileName(_filePath)}"; + + public JsonZstdFileSource(string filePath, JsonColumnSchema[] schema); + + public Task ReadDataAsync(CancellationToken ct = default); + public ValueTask DisposeAsync(); +} +``` + +### DevEtlRegistry + +Central registry for all development ETL pipelines: + +```csharp +public class DevEtlRegistry +{ + private readonly IDbConnectionFactory _factory; + private readonly string _cacheDirectory; + + public EtlPipeline GetPipeline(string tableName); + public IEnumerable GetAvailableTables(); + public async Task RunAsync(string tableName, CancellationToken ct); + public async Task> RunAllAsync(CancellationToken ct); +} +``` + +### Per-Table ETL Classes + +Each table has a static class with explicit schema (generated by reading SQL scripts): + +```csharp +public static class BranchDevEtl +{ + public static readonly string TableName = "Branch"; + public static readonly string CacheFileName = "branch.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = new[] + { + new JsonColumnSchema("Code", typeof(string)), + new JsonColumnSchema("Description", typeof(string)), + new JsonColumnSchema("LastUpdateDT", typeof(DateTime)), + }; + + public static EtlPipeline Create(IDbConnectionFactory factory, string cacheFilePath) + { + return new EtlPipelineBuilder() + .WithName("Branch_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(factory, "Branch")) + .Build(); + } +} +``` + +## File Organization + +``` +NEW/src/JdeScoping.DataSync/ +├── Etl/ +│ ├── Sources/ +│ │ ├── DbQuerySource.cs (existing) +│ │ ├── JsonZstdFileSource.cs (new) +│ │ └── JsonStreamingDataReader.cs (new) +│ └── Models/ +│ └── JsonColumnSchema.cs (new) +│ +├── DevEtl/ +│ ├── DevEtlRegistry.cs (new) +│ ├── BranchDevEtl.cs (new) +│ ├── OrgHierarchyDevEtl.cs (new) +│ ├── WorkCenterDevEtl.cs (new) +│ ├── ProfitCenterDevEtl.cs (new) +│ ├── JdeUserDevEtl.cs (new) +│ ├── ItemDevEtl.cs (new) +│ ├── LotDevEtl.cs (new) +│ ├── FunctionCodeDevEtl.cs (new) +│ ├── RouteMasterDevEtl.cs (new) +│ ├── MisDataDevEtl.cs (new) +│ ├── WorkOrderCurrDevEtl.cs (new) +│ ├── WorkOrderHistDevEtl.cs (new) +│ ├── LotUsageCurrDevEtl.cs (new) +│ ├── LotUsageHistDevEtl.cs (new) +│ ├── WorkOrderTimeCurrDevEtl.cs (new) +│ ├── WorkOrderTimeHistDevEtl.cs (new) +│ ├── WorkOrderStepCurrDevEtl.cs (new) +│ ├── WorkOrderStepHistDevEtl.cs (new) +│ ├── WorkOrderComponentCurrDevEtl.cs (new) +│ ├── WorkOrderComponentHistDevEtl.cs (new) +│ └── WorkOrderRoutingDevEtl.cs (new) +``` + +## Dependencies + +**New NuGet Package:** +- `ZstdSharp.Port` - Pure C# zstd decompression (no native dependencies) + +## SQL Type to CLR Type Mapping + +| SQL Type | CLR Type | +|----------|----------| +| `VARCHAR(n)`, `NVARCHAR(n)` | `string` | +| `INT` | `int` | +| `BIGINT` | `long` | +| `DECIMAL(p,s)`, `NUMERIC(p,s)` | `decimal` | +| `DATETIME`, `DATETIME2(n)` | `DateTime` | +| `BIT` | `bool` | +| `VARBINARY(n)` | `byte[]` | + +## Cache File Inventory + +| Table | Cache File | Size | +|-------|------------|------| +| Branch | branch.json.zstd | 930 B | +| OrgHierarchy | orghierarchy.json.zstd | 36 KB | +| WorkCenter | workcenter.json.zstd | 65 KB | +| ProfitCenter | profitcenter.json.zstd | 148 KB | +| JdeUser | jdeuser.json.zstd | 2.4 MB | +| FunctionCode | functioncode.json.zstd | 3.2 MB | +| Item | item.json.zstd | 17 MB | +| RouteMaster | routemaster.json.zstd | 20 MB | +| WorkOrder_Hist | workorder_hist.json.zstd | 41 MB | +| WorkOrder_Curr | workorder_curr.json.zstd | 86 MB | +| LotUsage_Hist | lotusage_hist.json.zstd | 146 MB | +| WorkOrderComponent_Hist | workordercomponent_hist.json.zstd | 148 MB | +| Lot | lot.json.zstd | 184 MB | +| MisData | misdata.json.zstd | 178 MB | +| WorkOrderStep_Hist | workorderstep_hist.json.zstd | 268 MB | +| WorkOrderComponent_Curr | workordercomponent_curr.json.zstd | 314 MB | +| WorkOrderRouting | workorderrouting.json.zstd | 324 MB | +| LotUsage_Curr | lotusage_curr.json.zstd | 400 MB | +| WorkOrderStep_Curr | workorderstep_curr.json.zstd | 507 MB | +| WorkOrderTime_Hist | workordertime_hist.json.zstd | 512 MB | +| WorkOrderTime_Curr | workordertime_curr.json.zstd | 879 MB | + +**Note:** StatusCode has no cache file. + +## Memory Considerations + +The streaming approach ensures: +- Only one JSON object in memory at a time (~1-10 KB per row) +- Decompression buffer ~64 KB +- Suitable for all file sizes including 879 MB workordertime_curr + +## Testing Strategy + +1. Unit tests for `JsonStreamingDataReader` with small JSON samples +2. Integration test loading Branch (smallest) to validate end-to-end +3. Integration test loading WorkOrderTime_Curr (largest) to validate streaming diff --git a/PLANS/2026-01-03-dev-etl-pipeline-implementation.md b/PLANS/2026-01-03-dev-etl-pipeline-implementation.md new file mode 100644 index 0000000..5148d1f --- /dev/null +++ b/PLANS/2026-01-03-dev-etl-pipeline-implementation.md @@ -0,0 +1,868 @@ +# Development ETL Pipeline Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Create development ETL pipelines that load cached `.json.zstd` files into SQL Server for local development. + +**Architecture:** Streaming JSON reader (`JsonZstdFileSource`) feeds into existing ETL pipeline infrastructure. + +**Tech Stack:** .NET 10, ZstdSharp, System.Text.Json, existing ETL framework + +--- + +## Phase 1: Core Infrastructure + Branch Table + +### Task 1: Add ZstdSharp NuGet Package + +**Files:** +- Modify: `NEW/src/JdeScoping.DataSync/JdeScoping.DataSync.csproj` + +**Step 1: Add package reference** + +```xml + +``` + +**Step 2: Verify package restores** + +Run: `dotnet restore NEW/src/JdeScoping.DataSync/JdeScoping.DataSync.csproj` +Expected: Restore succeeds + +--- + +### Task 2: Create JsonColumnSchema + +**Files:** +- Create: `NEW/src/JdeScoping.DataSync/Etl/Models/JsonColumnSchema.cs` + +**Step 1: Create the file** + +```csharp +namespace JdeScoping.DataSync.Etl.Models; + +/// +/// Defines a column schema for JSON-to-DataReader mapping. +/// +public record JsonColumnSchema( + string Name, + Type ClrType, + bool IsNullable = true) +{ + /// + /// Gets the SQL type name for this column (used in error messages). + /// + public string SqlTypeName => ClrType switch + { + Type t when t == typeof(string) => "VARCHAR", + Type t when t == typeof(int) => "INT", + Type t when t == typeof(long) => "BIGINT", + Type t when t == typeof(decimal) => "DECIMAL", + Type t when t == typeof(DateTime) => "DATETIME2", + Type t when t == typeof(bool) => "BIT", + Type t when t == typeof(byte[]) => "VARBINARY", + _ => "UNKNOWN" + }; +} +``` + +**Step 2: Verify it compiles** + +Run: `dotnet build NEW/src/JdeScoping.DataSync/JdeScoping.DataSync.csproj` +Expected: Build succeeds + +--- + +### Task 3: Create JsonStreamingDataReader + +**Files:** +- Create: `NEW/src/JdeScoping.DataSync/Etl/Sources/JsonStreamingDataReader.cs` + +**Step 1: Create the file** + +```csharp +using System.Data; +using System.Text.Json; +using JdeScoping.DataSync.Etl.Models; + +namespace JdeScoping.DataSync.Etl.Sources; + +/// +/// Streams a JSON array as an IDataReader, parsing one object at a time. +/// +internal sealed class JsonStreamingDataReader : IDataReader +{ + private readonly Stream _stream; + private readonly StreamReader _streamReader; + private readonly JsonColumnSchema[] _schema; + private readonly Dictionary _nameToOrdinal; + private object?[] _currentRow; + private bool _disposed; + private bool _started; + private bool _finished; + + public JsonStreamingDataReader(Stream stream, JsonColumnSchema[] schema) + { + _stream = stream ?? throw new ArgumentNullException(nameof(stream)); + _schema = schema ?? throw new ArgumentNullException(nameof(schema)); + _streamReader = new StreamReader(stream); + _currentRow = new object?[schema.Length]; + + _nameToOrdinal = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 0; i < schema.Length; i++) + { + _nameToOrdinal[schema[i].Name] = i; + } + } + + public int FieldCount => _schema.Length; + public int Depth => 0; + public bool IsClosed => _disposed; + public int RecordsAffected => -1; + + public object this[int ordinal] => GetValue(ordinal); + public object this[string name] => GetValue(GetOrdinal(name)); + + public string GetName(int ordinal) => _schema[ordinal].Name; + public int GetOrdinal(string name) => _nameToOrdinal.TryGetValue(name, out var ordinal) + ? ordinal + : throw new IndexOutOfRangeException($"Column '{name}' not found."); + + public Type GetFieldType(int ordinal) => _schema[ordinal].ClrType; + public string GetDataTypeName(int ordinal) => _schema[ordinal].SqlTypeName; + + public object GetValue(int ordinal) => _currentRow[ordinal] ?? DBNull.Value; + public bool IsDBNull(int ordinal) => _currentRow[ordinal] is null; + + public bool Read() + { + if (_disposed || _finished) return false; + + try + { + // Skip to start of array on first read + if (!_started) + { + SkipWhitespaceAndExpect('['); + _started = true; + } + + // Check for end of array or next object + SkipWhitespace(); + var next = (char)_streamReader.Peek(); + + if (next == ']') + { + _finished = true; + return false; + } + + if (next == ',') + { + _streamReader.Read(); // consume comma + SkipWhitespace(); + } + + // Read the next JSON object + var jsonObject = ReadJsonObject(); + if (jsonObject == null) + { + _finished = true; + return false; + } + + // Map JSON properties to row + Array.Clear(_currentRow); + foreach (var property in jsonObject.RootElement.EnumerateObject()) + { + if (_nameToOrdinal.TryGetValue(property.Name, out var ordinal)) + { + _currentRow[ordinal] = ParseValue(property.Value, _schema[ordinal].ClrType); + } + } + + return true; + } + catch (JsonException ex) + { + throw new InvalidDataException($"Failed to parse JSON: {ex.Message}", ex); + } + } + + private JsonDocument? ReadJsonObject() + { + SkipWhitespace(); + if (_streamReader.Peek() == -1 || (char)_streamReader.Peek() == ']') + return null; + + // Read characters until we have a complete JSON object + var buffer = new System.Text.StringBuilder(); + int braceCount = 0; + bool inString = false; + bool escaped = false; + + while (true) + { + int c = _streamReader.Read(); + if (c == -1) break; + + char ch = (char)c; + buffer.Append(ch); + + if (escaped) + { + escaped = false; + continue; + } + + if (ch == '\\' && inString) + { + escaped = true; + continue; + } + + if (ch == '"') + { + inString = !inString; + continue; + } + + if (!inString) + { + if (ch == '{') braceCount++; + else if (ch == '}') + { + braceCount--; + if (braceCount == 0) break; + } + } + } + + var json = buffer.ToString().Trim(); + if (string.IsNullOrEmpty(json) || json == "]") + return null; + + return JsonDocument.Parse(json); + } + + private static object? ParseValue(JsonElement element, Type targetType) + { + if (element.ValueKind == JsonValueKind.Null) + return null; + + if (targetType == typeof(string)) + return element.GetString(); + + if (targetType == typeof(int)) + return element.TryGetInt32(out var i) ? i : (int)element.GetDouble(); + + if (targetType == typeof(long)) + return element.TryGetInt64(out var l) ? l : (long)element.GetDouble(); + + if (targetType == typeof(decimal)) + return element.TryGetDecimal(out var d) ? d : (decimal)element.GetDouble(); + + if (targetType == typeof(DateTime)) + { + if (element.ValueKind == JsonValueKind.String) + return DateTime.Parse(element.GetString()!, null, System.Globalization.DateTimeStyles.RoundtripKind); + return element.GetDateTime(); + } + + if (targetType == typeof(bool)) + return element.GetBoolean(); + + if (targetType == typeof(byte[])) + return element.GetBytesFromBase64(); + + if (targetType == typeof(double)) + return element.GetDouble(); + + throw new NotSupportedException($"Type {targetType.Name} is not supported."); + } + + private void SkipWhitespace() + { + while (_streamReader.Peek() != -1 && char.IsWhiteSpace((char)_streamReader.Peek())) + { + _streamReader.Read(); + } + } + + private void SkipWhitespaceAndExpect(char expected) + { + SkipWhitespace(); + var actual = (char)_streamReader.Read(); + if (actual != expected) + throw new InvalidDataException($"Expected '{expected}' but found '{actual}'."); + } + + // IDataReader methods - typed getters + public bool GetBoolean(int ordinal) => (bool)GetValue(ordinal); + public byte GetByte(int ordinal) => (byte)GetValue(ordinal); + public long GetBytes(int ordinal, long fieldOffset, byte[]? buffer, int bufferOffset, int length) + { + var data = (byte[])GetValue(ordinal); + if (buffer == null) return data.Length; + var toCopy = Math.Min(length, data.Length - (int)fieldOffset); + Array.Copy(data, fieldOffset, buffer, bufferOffset, toCopy); + return toCopy; + } + public char GetChar(int ordinal) => ((string)GetValue(ordinal))[0]; + public long GetChars(int ordinal, long fieldOffset, char[]? buffer, int bufferOffset, int length) + { + var data = (string)GetValue(ordinal); + if (buffer == null) return data.Length; + var toCopy = Math.Min(length, data.Length - (int)fieldOffset); + data.CopyTo((int)fieldOffset, buffer, bufferOffset, toCopy); + return toCopy; + } + public IDataReader GetData(int ordinal) => throw new NotSupportedException(); + public DateTime GetDateTime(int ordinal) => (DateTime)GetValue(ordinal); + public decimal GetDecimal(int ordinal) => (decimal)GetValue(ordinal); + public double GetDouble(int ordinal) => (double)GetValue(ordinal); + public float GetFloat(int ordinal) => (float)GetValue(ordinal); + public Guid GetGuid(int ordinal) => (Guid)GetValue(ordinal); + public short GetInt16(int ordinal) => (short)GetValue(ordinal); + public int GetInt32(int ordinal) => (int)GetValue(ordinal); + public long GetInt64(int ordinal) => (long)GetValue(ordinal); + public string GetString(int ordinal) => (string)GetValue(ordinal); + public int GetValues(object[] values) + { + var count = Math.Min(values.Length, _currentRow.Length); + for (int i = 0; i < count; i++) + values[i] = GetValue(i); + return count; + } + + public DataTable GetSchemaTable() + { + var table = new DataTable("SchemaTable"); + table.Columns.Add("ColumnName", typeof(string)); + table.Columns.Add("ColumnOrdinal", typeof(int)); + table.Columns.Add("DataType", typeof(Type)); + table.Columns.Add("AllowDBNull", typeof(bool)); + + for (int i = 0; i < _schema.Length; i++) + { + table.Rows.Add(_schema[i].Name, i, _schema[i].ClrType, _schema[i].IsNullable); + } + + return table; + } + + public bool NextResult() => false; + + public void Close() => Dispose(); + + public void Dispose() + { + if (!_disposed) + { + _streamReader.Dispose(); + _disposed = true; + } + } +} +``` + +**Step 2: Verify it compiles** + +Run: `dotnet build NEW/src/JdeScoping.DataSync/JdeScoping.DataSync.csproj` +Expected: Build succeeds + +--- + +### Task 4: Create JsonZstdFileSource + +**Files:** +- Create: `NEW/src/JdeScoping.DataSync/Etl/Sources/JsonZstdFileSource.cs` + +**Step 1: Create the file** + +```csharp +using System.Data; +using JdeScoping.DataSync.Etl.Contracts; +using JdeScoping.DataSync.Etl.Models; +using ZstdSharp; + +namespace JdeScoping.DataSync.Etl.Sources; + +/// +/// Import source that reads from a zstd-compressed JSON array file. +/// +public sealed class JsonZstdFileSource : IImportSource +{ + private readonly string _filePath; + private readonly JsonColumnSchema[] _schema; + private FileStream? _fileStream; + private DecompressionStream? _decompressionStream; + private JsonStreamingDataReader? _reader; + + public string SourceName => $"JsonZstd:{Path.GetFileName(_filePath)}"; + + public JsonZstdFileSource(string filePath, JsonColumnSchema[] schema) + { + if (string.IsNullOrWhiteSpace(filePath)) + throw new ArgumentException("File path cannot be null or empty.", nameof(filePath)); + + if (!File.Exists(filePath)) + throw new FileNotFoundException($"Cache file not found: {filePath}", filePath); + + _filePath = filePath; + _schema = schema ?? throw new ArgumentNullException(nameof(schema)); + } + + public Task ReadDataAsync(CancellationToken cancellationToken = default) + { + _fileStream = new FileStream(_filePath, FileMode.Open, FileAccess.Read, FileShare.Read, + bufferSize: 65536, useAsync: true); + _decompressionStream = new DecompressionStream(_fileStream); + _reader = new JsonStreamingDataReader(_decompressionStream, _schema); + + return Task.FromResult(_reader); + } + + public async ValueTask DisposeAsync() + { + if (_reader != null) + { + _reader.Dispose(); + _reader = null; + } + + if (_decompressionStream != null) + { + await _decompressionStream.DisposeAsync(); + _decompressionStream = null; + } + + if (_fileStream != null) + { + await _fileStream.DisposeAsync(); + _fileStream = null; + } + } +} +``` + +**Step 2: Verify it compiles** + +Run: `dotnet build NEW/src/JdeScoping.DataSync/JdeScoping.DataSync.csproj` +Expected: Build succeeds + +--- + +### Task 5: Create BranchDevEtl + +**Files:** +- Create: `NEW/src/JdeScoping.DataSync/DevEtl/BranchDevEtl.cs` + +**Reference - Branch table schema from `003_CreateBranchTable.sql`:** +- `Code` VARCHAR(12) NOT NULL +- `Description` VARCHAR(40) NULL +- `LastUpdateDT` DATETIME2(7) NOT NULL + +**Step 1: Create the file** + +```csharp +using JdeScoping.DataAccess; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.DevEtl; + +/// +/// Development ETL pipeline for the Branch table. +/// +public static class BranchDevEtl +{ + public static readonly string TableName = "Branch"; + public static readonly string CacheFileName = "branch.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("Code", typeof(string), IsNullable: false), + new("Description", typeof(string), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} +``` + +**Step 2: Verify it compiles** + +Run: `dotnet build NEW/src/JdeScoping.DataSync/JdeScoping.DataSync.csproj` +Expected: Build succeeds + +--- + +### Task 6: Create DevEtlRegistry + +**Files:** +- Create: `NEW/src/JdeScoping.DataSync/DevEtl/DevEtlRegistry.cs` + +**Step 1: Create the file** + +```csharp +using JdeScoping.DataAccess; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Results; +using Microsoft.Extensions.Logging; + +namespace JdeScoping.DataSync.DevEtl; + +/// +/// Registry for development ETL pipelines that load from cached JSON files. +/// +public class DevEtlRegistry +{ + private readonly IDbConnectionFactory _connectionFactory; + private readonly string _cacheDirectory; + private readonly ILogger? _logger; + + private readonly Dictionary> _pipelineFactories = new(StringComparer.OrdinalIgnoreCase) + { + [BranchDevEtl.TableName] = (factory, cacheDir) => + BranchDevEtl.Create(factory, Path.Combine(cacheDir, BranchDevEtl.CacheFileName)), + }; + + public DevEtlRegistry( + IDbConnectionFactory connectionFactory, + string cacheDirectory, + ILogger? logger = null) + { + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + + if (string.IsNullOrWhiteSpace(cacheDirectory)) + throw new ArgumentException("Cache directory is required.", nameof(cacheDirectory)); + + if (!Directory.Exists(cacheDirectory)) + throw new DirectoryNotFoundException($"Cache directory not found: {cacheDirectory}"); + + _cacheDirectory = cacheDirectory; + _logger = logger; + } + + public IEnumerable GetAvailableTables() => _pipelineFactories.Keys; + + public EtlPipeline GetPipeline(string tableName) + { + if (!_pipelineFactories.TryGetValue(tableName, out var factory)) + throw new ArgumentException($"No pipeline registered for table '{tableName}'.", nameof(tableName)); + + return factory(_connectionFactory, _cacheDirectory); + } + + public async Task RunAsync(string tableName, CancellationToken cancellationToken = default) + { + _logger?.LogInformation("Running dev ETL for {TableName}", tableName); + + var pipeline = GetPipeline(tableName); + var result = await pipeline.ExecuteAsync(cancellationToken); + + if (result.Success) + _logger?.LogInformation("Completed {TableName}: {Rows} rows in {Elapsed:g}", + tableName, result.TotalRows, result.Elapsed); + else + _logger?.LogError(result.Error, "Failed {TableName}: {Error}", + tableName, result.Error?.Message); + + return result; + } + + public async Task> RunAllAsync(CancellationToken cancellationToken = default) + { + var results = new List(); + + foreach (var tableName in GetAvailableTables()) + { + if (cancellationToken.IsCancellationRequested) + break; + + var result = await RunAsync(tableName, cancellationToken); + results.Add(result); + } + + return results; + } +} +``` + +**Step 2: Verify it compiles** + +Run: `dotnet build NEW/src/JdeScoping.DataSync/JdeScoping.DataSync.csproj` +Expected: Build succeeds + +--- + +### Task 7: Create Integration Test for Branch + +**Files:** +- Create: `NEW/tests/JdeScoping.DataSync.Tests/DevEtl/BranchDevEtlTests.cs` + +**Step 1: Create the test file** + +```csharp +using FluentAssertions; +using JdeScoping.DataAccess; +using JdeScoping.DataSync.DevEtl; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; +using Xunit; + +namespace JdeScoping.DataSync.Tests.DevEtl; + +/// +/// Integration tests for Branch development ETL. +/// Requires: Local SQL Server, CACHED_DB_FILES directory with branch.json.zstd +/// +public class BranchDevEtlTests : IAsyncLifetime +{ + private readonly string _connectionString; + private readonly string _cacheDirectory; + private readonly IDbConnectionFactory _connectionFactory; + + public BranchDevEtlTests() + { + // Load configuration + var config = new ConfigurationBuilder() + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables() + .Build(); + + _connectionString = config.GetConnectionString("LotFinder") + ?? throw new InvalidOperationException("LotFinder connection string not configured."); + + _cacheDirectory = config["DevEtl:CacheDirectory"] + ?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES"); + + _connectionFactory = new DbConnectionFactory(_connectionString); + } + + public async Task InitializeAsync() + { + // Ensure Branch table is empty before test + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + await using var command = new SqlCommand("TRUNCATE TABLE dbo.Branch", connection); + await command.ExecuteNonQueryAsync(); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public async Task Create_ReturnsValidPipeline() + { + // Arrange + var cacheFilePath = Path.Combine(_cacheDirectory, BranchDevEtl.CacheFileName); + Skip.IfNot(File.Exists(cacheFilePath), $"Cache file not found: {cacheFilePath}"); + + // Act + var pipeline = BranchDevEtl.Create(_connectionFactory, cacheFilePath); + + // Assert + pipeline.Should().NotBeNull(); + pipeline.Name.Should().Be("Branch_Dev"); + } + + [Fact] + public async Task Execute_LoadsBranchData() + { + // Arrange + var cacheFilePath = Path.Combine(_cacheDirectory, BranchDevEtl.CacheFileName); + Skip.IfNot(File.Exists(cacheFilePath), $"Cache file not found: {cacheFilePath}"); + + var pipeline = BranchDevEtl.Create(_connectionFactory, cacheFilePath); + + // Act + var result = await pipeline.ExecuteAsync(); + + // Assert + result.Success.Should().BeTrue(because: result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.Should().BeGreaterThan(0, "Should load at least one row"); + + // Verify data in database + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + await using var command = new SqlCommand("SELECT COUNT(*) FROM dbo.Branch", connection); + var count = (int)(await command.ExecuteScalarAsync())!; + + count.Should().Be((int)result.TotalRows, "Database row count should match pipeline result"); + } + + [Fact] + public async Task Registry_RunAsync_LoadsBranch() + { + // Arrange + Skip.IfNot(Directory.Exists(_cacheDirectory), $"Cache directory not found: {_cacheDirectory}"); + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + + // Act + var result = await registry.RunAsync("Branch"); + + // Assert + result.Success.Should().BeTrue(because: result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.Should().BeGreaterThan(0); + } +} +``` + +**Step 2: Add test project dependencies if needed** + +Verify `JdeScoping.DataSync.Tests.csproj` has: +- Reference to `JdeScoping.DataSync` +- FluentAssertions +- xunit +- xunit.runner.visualstudio + +**Step 3: Run the tests** + +Run: `dotnet test NEW/tests/JdeScoping.DataSync.Tests --filter "FullyQualifiedName~BranchDevEtlTests"` +Expected: Tests pass (or skip if cache file not found) + +--- + +### Task 8: Run End-to-End Test and Debug + +**Step 1: Ensure database is running** + +Run: `docker ps | grep scopingtool-sqlserver` +Expected: Container is running + +**Step 2: Run the integration test** + +Run: `dotnet test NEW/tests/JdeScoping.DataSync.Tests --filter "BranchDevEtlTests.Execute_LoadsBranchData" -v normal` + +**Step 3: If test fails, debug the issue** + +Common issues to check: +- Connection string correct in appsettings.json +- Cache file exists and is readable +- Branch table exists in database +- JSON parsing errors (check column name case sensitivity) + +**Step 4: Verify data in database** + +Run SQL: `SELECT TOP 5 * FROM dbo.Branch ORDER BY Code` +Expected: See branch records from cache file + +--- + +## Phase 2: Lessons Learned + +### Issues Encountered and Fixes + +1. **JsonDocument Memory Leak** + - **Issue:** `ReadJsonObject()` returned `JsonDocument` that wasn't being disposed, causing memory accumulation + - **Fix:** Changed to `using var jsonObject = ReadJsonObject();` in the `Read()` method + - **Lesson:** Always dispose `JsonDocument` instances - they own native memory + +2. **Multiple ReadDataAsync Calls** + - **Issue:** `JsonZstdFileSource.ReadDataAsync()` could be called multiple times, causing stream leaks + - **Fix:** Added guard: `if (_fileStream != null) throw new InvalidOperationException(...)` + - **Lesson:** Sources should only be readable once; enforce this with guards + +3. **Exception Safety in Stream Initialization** + - **Issue:** If stream creation failed partway through (e.g., DecompressionStream fails), earlier streams leaked + - **Fix:** Wrapped initialization in try-catch with cleanup in catch block: + ```csharp + try { + _fileStream = new FileStream(...); + _decompressionStream = new DecompressionStream(_fileStream); + _reader = new JsonStreamingDataReader(...); + return Task.FromResult(_reader); + } catch { + _reader?.Dispose(); + _decompressionStream?.Dispose(); + _fileStream?.Dispose(); + throw; + } + ``` + - **Lesson:** Multi-resource initialization needs exception safety + +4. **Cancellation Token Handling** + - **Issue:** `RunAllAsync` used `IsCancellationRequested + break` which silently stops without exception + - **Fix:** Changed to `cancellationToken.ThrowIfCancellationRequested();` + - **Lesson:** Prefer `ThrowIfCancellationRequested()` for proper cancellation semantics + +5. **Connection String Naming Convention** + - **Issue:** Test used `"LotFinder"` but `DbConnectionFactory` expects `"LotFinderDB"` + - **Fix:** Updated appsettings.json key to `"LotFinderDB"` + - **Lesson:** Match connection string names to what `DbConnectionFactory` expects + +6. **Hardcoded Absolute Paths** + - **Issue:** Fallback cache directory path was user-specific `/Users/dohertj2/Desktop/...` + - **Fix:** Changed to relative path using `Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "...")` + - **Lesson:** Use relative paths for portability; config should specify absolute paths + +### Patterns That Worked Well + +1. **IAsyncLifetime for Test Isolation** + - Using `IAsyncLifetime.InitializeAsync()` to truncate tables before each test ensures clean state + - Pattern: `TRUNCATE TABLE dbo.{Table}` in `InitializeAsync()` + +2. **Shouldly Assertions** + - Project uses Shouldly instead of FluentAssertions + - Pattern: `result.Success.ShouldBeTrue(result.Error?.Message ?? "reason")` + +3. **Nullable File Checks in Tests** + - Early return when cache files don't exist (graceful skip) + - Pattern: `if (!File.Exists(cacheFilePath)) return;` + +4. **Static Factory Pattern for DevEtl Classes** + - Clean separation: static `Create()` method with explicit validation + - Pattern: `ArgumentNullException.ThrowIfNull(connectionFactory);` + +5. **Property Naming** + - Pipeline property is `PipelineName` (not `Name`) + - Pattern: `pipeline.PipelineName.ShouldBe("Branch_Dev")` + +### Performance Observations + +- Branch table (930 bytes compressed, ~10 rows) loads in ~75ms including decompression +- Streaming approach successfully processes one JSON object at a time +- No memory issues observed - suitable for larger files + +### Code Corrections from Original Plan + +| Original Plan | Actual Implementation | +|---------------|----------------------| +| `pipeline.Name` | `pipeline.PipelineName` | +| FluentAssertions | Shouldly | +| `Skip.IfNot()` | Early return with `if (!exists) return;` | +| `IDbConnectionFactory` constructor with string | Constructor takes `IConfiguration` | +| Dapper for test queries | Direct `SqlConnection` + `ExecuteScalarAsync` | + +--- + +## Phase 3: Remaining Tables + +After Phase 2, add remaining tables following the established pattern. Priority order by file size: + +1. **Small (< 1 MB):** OrgHierarchy, WorkCenter, ProfitCenter +2. **Medium (1-20 MB):** JdeUser, FunctionCode, Item, RouteMaster +3. **Large (20-200 MB):** Lot, MisData, WorkOrder_Curr/Hist, LotUsage_Hist +4. **Very Large (200+ MB):** LotUsage_Curr, WorkOrderRouting, WorkOrderStep, WorkOrderTime, WorkOrderComponent + +For each table: +1. Read the CREATE TABLE script from Database/Scripts/ +2. Create `{Table}DevEtl.cs` with explicit schema +3. Register in `DevEtlRegistry._pipelineFactories` +4. Add integration test +5. Verify with sample data diff --git a/PLANS/2026-01-03-etl-performance-optimization.md b/PLANS/2026-01-03-etl-performance-optimization.md new file mode 100644 index 0000000..bc4745f --- /dev/null +++ b/PLANS/2026-01-03-etl-performance-optimization.md @@ -0,0 +1,422 @@ +# ETL Performance Optimization Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Optimize dev ETL pipelines to significantly reduce load times for large tables (currently 18+ minutes for 88M rows) + +**Architecture:** Four-pronged optimization: SqlBulkCopy tuning, parallel table loading, Utf8JsonReader parsing, and zstd buffer optimization + +**Tech Stack:** .NET 10, System.Text.Json (Utf8JsonReader), System.Buffers (ArrayPool), SqlBulkCopy, ZstdSharp + +--- + +## Task 1: SqlBulkCopy Performance Tuning + +**Files:** +- Modify: `NEW/src/JdeScoping.DataSync/Etl/Destinations/DbBulkImportDestination.cs` + +**Step 1: Add SqlBulkCopyOptions.TableLock** + +TableLock acquires a bulk update lock on the table during the bulk copy operation, reducing logging overhead. + +```csharp +using var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.TableLock, null) +{ + DestinationTableName = qualifiedName, + BatchSize = _batchSize, + BulkCopyTimeout = _commandTimeoutSeconds, + EnableStreaming = true +}; +``` + +**Step 2: Increase default batch size** + +Change from 10,000 to 100,000 rows per batch to reduce round-trips. + +```csharp +private const int DefaultBatchSize = 100000; +``` + +**Step 3: Reduce NotifyAfter frequency** + +Currently fires 8,800+ times for 88M rows. Reduce to every 10 batches. + +```csharp +bulkCopy.NotifyAfter = _batchSize * 10; +``` + +**Step 4: Add option for infinite timeout** + +Add a constant for large table loads. + +```csharp +private const int InfiniteTimeout = 0; +``` + +**Step 5: Verify changes compile** + +Run: `dotnet build NEW/src/JdeScoping.DataSync/` +Expected: Build succeeded + +--- + +## Task 2: Parallel Loading in DevEtlRegistry + +**Files:** +- Modify: `NEW/src/JdeScoping.DataSync/DevEtl/DevEtlRegistry.cs` + +**Step 1: Add RunAllParallelAsync method** + +```csharp +public async Task> RunAllParallelAsync( + int maxDegreeOfParallelism = 4, + CancellationToken cancellationToken = default) +{ + var results = new ConcurrentBag(); + var semaphore = new SemaphoreSlim(maxDegreeOfParallelism); + + // Separate tables by size - run very large ones sequentially at the end + var smallMediumTables = GetAvailableTables() + .Where(t => !IsVeryLargeTable(t)) + .ToList(); + var veryLargeTables = GetAvailableTables() + .Where(IsVeryLargeTable) + .ToList(); + + // Run small/medium tables in parallel + var tasks = smallMediumTables.Select(async tableName => + { + await semaphore.WaitAsync(cancellationToken); + try + { + var result = await RunAsync(tableName, cancellationToken); + results.Add(result); + } + finally + { + semaphore.Release(); + } + }); + + await Task.WhenAll(tasks); + + // Run very large tables sequentially (IO-bound, would contend) + foreach (var tableName in veryLargeTables) + { + cancellationToken.ThrowIfCancellationRequested(); + var result = await RunAsync(tableName, cancellationToken); + results.Add(result); + } + + return results.ToList(); +} + +private static bool IsVeryLargeTable(string tableName) => + tableName.Contains("WorkOrderTime", StringComparison.OrdinalIgnoreCase) || + tableName.Contains("WorkOrderStep", StringComparison.OrdinalIgnoreCase) || + tableName.Contains("WorkOrderRouting", StringComparison.OrdinalIgnoreCase); +``` + +**Step 2: Add required using statements** + +```csharp +using System.Collections.Concurrent; +``` + +**Step 3: Verify changes compile** + +Run: `dotnet build NEW/src/JdeScoping.DataSync/` +Expected: Build succeeded + +--- + +## Task 3: Utf8JsonReader-Based Streaming Parser + +**Files:** +- Create: `NEW/src/JdeScoping.DataSync/Etl/Sources/Utf8JsonStreamingDataReader.cs` +- Modify: `NEW/src/JdeScoping.DataSync/Etl/Sources/JsonZstdFileSource.cs` + +**Step 1: Create Utf8JsonStreamingDataReader** + +This replaces the current char-by-char parsing with efficient Utf8JsonReader. + +```csharp +using System.Buffers; +using System.Data; +using System.Text; +using System.Text.Json; +using JdeScoping.DataSync.Etl.Models; + +namespace JdeScoping.DataSync.Etl.Sources; + +/// +/// High-performance streaming JSON array reader using Utf8JsonReader. +/// +internal sealed class Utf8JsonStreamingDataReader : IDataReader +{ + private const int DefaultBufferSize = 256 * 1024; // 256 KB + + private readonly Stream _stream; + private readonly JsonColumnSchema[] _schema; + private readonly Dictionary _nameToOrdinal; + private readonly byte[][] _encodedColumnNames; + private byte[] _buffer; + private int _bytesInBuffer; + private int _bytesConsumed; + private JsonReaderState _readerState; + private object?[] _currentRow; + private bool _disposed; + private bool _started; + private bool _finished; + + public Utf8JsonStreamingDataReader(Stream stream, JsonColumnSchema[] schema) + { + _stream = stream ?? throw new ArgumentNullException(nameof(stream)); + _schema = schema ?? throw new ArgumentNullException(nameof(schema)); + _buffer = ArrayPool.Shared.Rent(DefaultBufferSize); + _currentRow = new object?[schema.Length]; + _readerState = new JsonReaderState(); + + _nameToOrdinal = new Dictionary(StringComparer.OrdinalIgnoreCase); + _encodedColumnNames = new byte[schema.Length][]; + for (int i = 0; i < schema.Length; i++) + { + _nameToOrdinal[schema[i].Name] = i; + _encodedColumnNames[i] = Encoding.UTF8.GetBytes(schema[i].Name); + } + } + + // ... IDataReader implementation +} +``` + +**Step 2: Implement Read() with Utf8JsonReader** + +Key difference: Parse directly from byte buffer, no string allocation per object. + +```csharp +public bool Read() +{ + if (_disposed || _finished) return false; + + try + { + while (true) + { + var reader = new Utf8JsonReader( + _buffer.AsSpan(_bytesConsumed, _bytesInBuffer - _bytesConsumed), + isFinalBlock: false, + _readerState); + + if (TryReadNextObject(ref reader)) + { + _bytesConsumed += (int)reader.BytesConsumed; + _readerState = reader.CurrentState; + return true; + } + + // Need more data + if (!RefillBuffer()) + { + // Final block + reader = new Utf8JsonReader( + _buffer.AsSpan(_bytesConsumed, _bytesInBuffer - _bytesConsumed), + isFinalBlock: true, + _readerState); + + if (TryReadNextObject(ref reader)) + { + _bytesConsumed += (int)reader.BytesConsumed; + return true; + } + + _finished = true; + return false; + } + } + } + catch (JsonException ex) + { + throw new InvalidDataException($"Failed to parse JSON: {ex.Message}", ex); + } +} +``` + +**Step 3: Implement TryReadNextObject with ValueTextEquals** + +Use pre-encoded column names to avoid string allocations. + +```csharp +private bool TryReadNextObject(ref Utf8JsonReader reader) +{ + if (!_started) + { + if (!reader.Read() || reader.TokenType != JsonTokenType.StartArray) + throw new InvalidDataException("Expected JSON array."); + _started = true; + } + + if (!reader.Read()) + return false; + + if (reader.TokenType == JsonTokenType.EndArray) + { + _finished = true; + return false; + } + + if (reader.TokenType != JsonTokenType.StartObject) + throw new InvalidDataException($"Expected object, got {reader.TokenType}"); + + Array.Clear(_currentRow); + + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) + { + if (reader.TokenType != JsonTokenType.PropertyName) + continue; + + // Find matching column using pre-encoded names + int ordinal = -1; + for (int i = 0; i < _encodedColumnNames.Length; i++) + { + if (reader.ValueTextEquals(_encodedColumnNames[i])) + { + ordinal = i; + break; + } + } + + if (!reader.Read()) + return false; + + if (ordinal >= 0) + { + _currentRow[ordinal] = ParseValue(ref reader, _schema[ordinal].ClrType); + } + } + + return reader.TokenType == JsonTokenType.EndObject; +} +``` + +**Step 4: Implement RefillBuffer** + +```csharp +private bool RefillBuffer() +{ + // Move unconsumed data to start of buffer + var remaining = _bytesInBuffer - _bytesConsumed; + if (remaining > 0) + { + Buffer.BlockCopy(_buffer, _bytesConsumed, _buffer, 0, remaining); + } + _bytesInBuffer = remaining; + _bytesConsumed = 0; + + // Read more data + var bytesRead = _stream.Read(_buffer, _bytesInBuffer, _buffer.Length - _bytesInBuffer); + if (bytesRead == 0) + return false; + + _bytesInBuffer += bytesRead; + return true; +} +``` + +**Step 5: Update JsonZstdFileSource to use new reader** + +Add constructor parameter to select reader implementation. + +```csharp +public JsonZstdFileSource(string filePath, JsonColumnSchema[] schema, bool useHighPerformanceReader = true) +{ + // ... existing validation ... + _useHighPerformanceReader = useHighPerformanceReader; +} + +public Task ReadDataAsync(CancellationToken cancellationToken = default) +{ + // ... existing setup ... + + _reader = _useHighPerformanceReader + ? new Utf8JsonStreamingDataReader(bufferedStream, _schema) + : new JsonStreamingDataReader(bufferedStream, _schema); + + return Task.FromResult(_reader); +} +``` + +**Step 6: Verify changes compile** + +Run: `dotnet build NEW/src/JdeScoping.DataSync/` +Expected: Build succeeded + +--- + +## Task 4: Zstd Buffer Optimizations + +**Files:** +- Modify: `NEW/src/JdeScoping.DataSync/Etl/Sources/JsonZstdFileSource.cs` + +**Step 1: Add SequentialScan and sync IO** + +```csharp +_fileStream = new FileStream( + _filePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 256 * 1024, // 256 KB + FileOptions.SequentialScan); // Hint for OS read-ahead +``` + +**Step 2: Wrap DecompressionStream in BufferedStream** + +```csharp +_decompressionStream = new DecompressionStream(_fileStream); +var bufferedStream = new BufferedStream(_decompressionStream, 256 * 1024); +_reader = new Utf8JsonStreamingDataReader(bufferedStream, _schema); +``` + +**Step 3: Verify changes compile** + +Run: `dotnet build NEW/src/JdeScoping.DataSync/` +Expected: Build succeeded + +--- + +## Task 5: Integration Testing + +**Files:** +- Test: `NEW/tests/JdeScoping.DataSync.Tests/DevEtl/` + +**Step 1: Run existing tests to ensure no regressions** + +Run: `dotnet test NEW/tests/JdeScoping.DataSync.Tests/ --filter "DevEtl"` +Expected: All tests pass + +**Step 2: Test parallel loading** + +Create a simple console app to test: +```csharp +var registry = new DevEtlRegistry(factory, cacheDir); +var sw = Stopwatch.StartNew(); +var results = await registry.RunAllParallelAsync(maxDegreeOfParallelism: 4); +Console.WriteLine($"Total time: {sw.Elapsed}"); +foreach (var r in results.OrderByDescending(r => r.TotalRows)) + Console.WriteLine($"{r.SourceName}: {r.TotalRows:N0} rows in {r.Elapsed}"); +``` + +--- + +## Expected Performance Improvements + +| Optimization | Expected Impact | +|--------------|-----------------| +| TableLock | 20-40% faster for large tables (reduced logging) | +| Batch size 100k | 10-20% faster (fewer round-trips) | +| Utf8JsonReader | 30-50% faster parsing (zero-alloc) | +| Parallel loading | 2-3x faster for full load (4 parallel) | +| Buffer optimizations | 5-10% faster IO | + +**Combined estimate:** Full load should drop from ~45 minutes to ~15-20 minutes. diff --git a/PLANS/2026-01-06-api-client-contracts-design.md b/PLANS/2026-01-06-api-client-contracts-design.md new file mode 100644 index 0000000..1a86bad --- /dev/null +++ b/PLANS/2026-01-06-api-client-contracts-design.md @@ -0,0 +1,781 @@ +# API Client Contracts Design + +**Date:** 2026-01-06 +**Status:** Approved (Revised after Codex review) + +## Purpose + +Define shared API contracts in `JdeScoping.Core` that ensure compile-time safety for: +- URL routes +- Request parameters +- Return types + +## Design Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Contract location | Core project | Both Api and Client already reference Core | +| Route handling | `ApiRoutes` static class with constants | Usable in `[HttpGet]` attributes AND client code | +| Result type | `OneOf` | Explicit error handling, preserves context | +| Controller returns | `ActionResult` with `ApiResult` mapper | Proper HTTP status codes | +| Client returns | `ApiResult` | Type-safe discriminated union | +| Error types | Mix (empty markers + detailed types) | Minimum info needed per case | +| CancellationToken | Optional with default (`ct = default`) | Clean call sites, Blazor-friendly | +| File contracts | Separate server/client interfaces | `IFormFile` vs `Stream` incompatibility | + +## Project Structure + +### New Files in Core + +``` +JdeScoping.Core/ +├── ApiContracts/ +│ ├── ApiRoutes.cs # Shared route constants +│ ├── ISearchApiClient.cs # Client contract +│ ├── ILookupApiClient.cs +│ ├── IAuthApiClient.cs +│ └── IFileApiClient.cs +├── ApiContracts/Results/ +│ ├── ApiResult.cs +│ ├── NotFound.cs +│ ├── Unauthorized.cs +│ ├── Forbidden.cs +│ ├── ValidationError.cs +│ ├── ApiError.cs +│ └── Unit.cs +``` + +### New Dependency in Core + +```xml + +``` + +## Route Constants + +### ApiRoutes.cs + +Using constants allows usage in both `[HttpGet]` attributes and client code: + +```csharp +namespace JdeScoping.Core.ApiContracts; + +/// +/// Shared API route constants. Use in controller attributes and client implementations. +/// +public static class ApiRoutes +{ + public static class Search + { + public const string Base = "api/search"; + public const string Queue = "api/search/queue"; + public const string ById = "api/search/{id:int}"; + public const string Copy = "api/search/{id:int}/copy"; + public const string Results = "api/search/{id:int}/results"; + + // Client route builders (handle parameter substitution) + public static string GetById(int id) => $"api/search/{id}"; + public static string GetCopy(int id) => $"api/search/{id}/copy"; + public static string GetResults(int id) => $"api/search/{id}/results"; + } + + public static class Lookup + { + public const string Items = "api/lookup/items"; + public const string ProfitCenters = "api/lookup/profit-centers"; + public const string WorkCenters = "api/lookup/work-centers"; + public const string Operators = "api/lookup/operators"; + + // Client route builders (handle URL encoding) + public static string FindItems(string query) => $"{Items}?q={Uri.EscapeDataString(query)}"; + public static string FindProfitCenters(string query) => $"{ProfitCenters}?q={Uri.EscapeDataString(query)}"; + public static string FindWorkCenters(string query) => $"{WorkCenters}?q={Uri.EscapeDataString(query)}"; + public static string FindOperators(string query) => $"{Operators}?q={Uri.EscapeDataString(query)}"; + } + + public static class Auth + { + public const string Base = "api/auth"; + public const string PublicKey = "api/auth/public-key"; + public const string Login = "api/auth/login"; + public const string Logout = "api/auth/logout"; + public const string Me = "api/auth/me"; + } + + public static class FileIO + { + public const string Base = "api/fileio"; + + // Downloads + public const string DownloadWorkOrders = "api/fileio/workorders/download"; + public const string DownloadItems = "api/fileio/items/download"; + public const string DownloadComponentLots = "api/fileio/componentlots/download"; + public const string DownloadPartOperations = "api/fileio/partoperations/download"; + + // Uploads + public const string UploadWorkOrders = "api/fileio/workorders/upload"; + public const string UploadItems = "api/fileio/items/upload"; + public const string UploadComponentLots = "api/fileio/componentlots/upload"; + public const string UploadPartOperations = "api/fileio/partoperations/upload"; + } +} +``` + +## Result Types + +### ApiResult.cs + +```csharp +using OneOf; + +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Standard API result type for client-side operations. +/// +[GenerateOneOf] +public partial class ApiResult : OneOfBase +{ + public bool IsSuccess => IsT0; + public bool IsNotFound => IsT1; + public bool IsValidationError => IsT2; + public bool IsUnauthorized => IsT3; + public bool IsForbidden => IsT4; + public bool IsError => IsT5; + + public T Value => AsT0; + public ValidationError ValidationError => AsT2; + public ApiError Error => AsT5; +} +``` + +### Error Types + +```csharp +namespace JdeScoping.Core.ApiContracts.Results; + +/// Resource not found (404). +public readonly record struct NotFound; + +/// Authentication required (401). +public readonly record struct Unauthorized; + +/// Access denied (403). +public readonly record struct Forbidden; + +/// +/// Validation failed (400) with field-level errors. +/// Maps to ASP.NET Core ProblemDetails format. +/// +public readonly record struct ValidationError(IReadOnlyDictionary FieldErrors) +{ + public static ValidationError FromProblemDetails(Dictionary errors) + => new(errors); +} + +/// General API error. +public readonly record struct ApiError(string Message, int? StatusCode = null); + +/// Empty success type for void operations. +public readonly record struct Unit; +``` + +## Client Interface Definitions + +Client interfaces define the contract for HTTP client implementations. They return `ApiResult`. + +### ISearchApiClient.cs + +```csharp +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for search API operations. +/// +public interface ISearchApiClient +{ + Task>> GetUserSearchesAsync(CancellationToken ct = default); + Task>> GetQueuedSearchesAsync(CancellationToken ct = default); + Task> GetSearchAsync(int id, CancellationToken ct = default); + Task> CopySearchAsync(int id, CancellationToken ct = default); + Task> CreateSearchAsync(SearchViewModel search, CancellationToken ct = default); + Task> GetResultsAsync(int id, CancellationToken ct = default); +} +``` + +### ILookupApiClient.cs + +```csharp +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for lookup/autocomplete API operations. +/// +public interface ILookupApiClient +{ + Task>> FindItemsAsync(string query, CancellationToken ct = default); + Task>> FindProfitCentersAsync(string query, CancellationToken ct = default); + Task>> FindWorkCentersAsync(string query, CancellationToken ct = default); + Task>> FindOperatorsAsync(string query, CancellationToken ct = default); +} +``` + +### IAuthApiClient.cs + +```csharp +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.Models; +using JdeScoping.Core.Models.Auth; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for authentication API operations. +/// +public interface IAuthApiClient +{ + Task> GetPublicKeyAsync(CancellationToken ct = default); + Task> LoginAsync(EncryptedLoginRequest request, CancellationToken ct = default); + Task> LogoutAsync(CancellationToken ct = default); + Task> GetCurrentUserAsync(CancellationToken ct = default); +} +``` + +### IFileApiClient.cs + +```csharp +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for file upload/download API operations. +/// Note: Uses Stream for client-side; controllers use IFormFile. +/// +public interface IFileApiClient +{ + // Downloads (POST with existing data, returns Excel bytes) + Task> DownloadWorkOrdersTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + Task> DownloadItemsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + Task> DownloadComponentLotsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + Task> DownloadPartOperationsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + + // Uploads (multipart form, returns parsed data) + Task>> UploadWorkOrdersAsync(Stream fileStream, string fileName, CancellationToken ct = default); + Task>> UploadItemsAsync(Stream fileStream, string fileName, CancellationToken ct = default); + Task>> UploadComponentLotsAsync(Stream fileStream, string fileName, CancellationToken ct = default); + Task>> UploadPartOperationsAsync(Stream fileStream, string fileName, CancellationToken ct = default); +} +``` + +## Controller Implementation + +Controllers use `ApiRoutes` constants in attributes and return `ActionResult`. A helper extension converts `ApiResult` to proper HTTP responses. + +### ApiResultExtensions.cs (in Api project) + +```csharp +using JdeScoping.Core.ApiContracts.Results; +using Microsoft.AspNetCore.Mvc; + +namespace JdeScoping.Api.Extensions; + +/// +/// Converts ApiResult to ActionResult with proper HTTP status codes. +/// +public static class ApiResultExtensions +{ + public static ActionResult ToActionResult(this ApiResult result) + { + return result.Match>( + success => new OkObjectResult(success), + notFound => new NotFoundResult(), + validation => new BadRequestObjectResult(new ValidationProblemDetails( + validation.FieldErrors.ToDictionary(k => k.Key, v => v.Value))), + unauthorized => new UnauthorizedResult(), + forbidden => new ForbidResult(), + error => new ObjectResult(new ProblemDetails + { + Status = error.StatusCode ?? 500, + Detail = error.Message + }) { StatusCode = error.StatusCode ?? 500 } + ); + } + + public static ActionResult ToCreatedResult(this ApiResult result, string actionName, Func routeValues) + { + return result.Match>( + success => new CreatedAtActionResult(actionName, null, routeValues(success), success), + notFound => new NotFoundResult(), + validation => new BadRequestObjectResult(new ValidationProblemDetails( + validation.FieldErrors.ToDictionary(k => k.Key, v => v.Value))), + unauthorized => new UnauthorizedResult(), + forbidden => new ForbidResult(), + error => new ObjectResult(new ProblemDetails + { + Status = error.StatusCode ?? 500, + Detail = error.Message + }) { StatusCode = error.StatusCode ?? 500 } + ); + } +} +``` + +### SearchController.cs + +```csharp +using JdeScoping.Api.Extensions; +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.Interfaces; +using JdeScoping.Core.ViewModels; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace JdeScoping.Api.Controllers; + +[Route(ApiRoutes.Search.Base)] +[ApiController] +[Authorize] +public class SearchController : ApiControllerBase +{ + private readonly ILotFinderRepository _repository; + + public SearchController(ILotFinderRepository repository) + { + _repository = repository; + } + + [HttpGet] + [ProducesResponseType(typeof(IEnumerable), StatusCodes.Status200OK)] + public async Task>> GetSearches(CancellationToken ct) + { + var searches = await _repository.GetUserSearchesAsync(CurrentUserName!, ct); + var viewModels = searches + .OrderByDescending(s => s.StartDt) + .Select(s => new SearchViewModel(s)) + .ToList(); + + return Ok(viewModels); + } + + [HttpGet("queue")] + [ProducesResponseType(typeof(IEnumerable), StatusCodes.Status200OK)] + public async Task>> GetQueuedSearches(CancellationToken ct) + { + var searches = await _repository.GetQueuedSearchesAsync(ct); + var viewModels = searches.Select(s => new SearchViewModel(s)).ToList(); + return Ok(viewModels); + } + + [HttpGet("{id:int}")] + [ProducesResponseType(typeof(SearchViewModel), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public async Task> GetSearch(int id, CancellationToken ct) + { + var search = await _repository.GetSearchAsync(id, ct); + if (search is null) + return NotFound(); + + return Ok(new SearchViewModel(search)); + } + + [HttpGet("{id:int}/copy")] + [ProducesResponseType(typeof(SearchViewModel), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public async Task> CopySearch(int id, CancellationToken ct) + { + var original = await _repository.GetSearchAsync(id, ct); + if (original is null) + return NotFound(); + + var copy = new Search + { + Id = 0, + UserName = CurrentUserName!, + Name = original.Name, + Status = SearchStatus.New, + CriteriaJson = original.CriteriaJson + }; + + return Ok(new SearchViewModel(copy)); + } + + [HttpPost] + [ProducesResponseType(typeof(int), StatusCodes.Status201Created)] + public async Task> CreateSearch( + [FromBody] SearchViewModel viewModel, + CancellationToken ct) + { + var search = viewModel.ToEntity(); + search.UserName = CurrentUserName!; + + var searchId = await _repository.SubmitSearchAsync(search, ct); + + return CreatedAtAction(nameof(GetSearch), new { id = searchId }, searchId); + } + + [HttpGet("{id:int}/results")] + [ProducesResponseType(typeof(FileContentResult), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public async Task GetResults(int id, CancellationToken ct) + { + var data = await _repository.GetSearchResultsAsync(id, ct); + if (data is null || data.Length == 0) + return NotFound(); + + return File( + data, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "search_results.xlsx"); + } +} +``` + +## Client Implementation + +### ApiClientBase.cs + +Shared HTTP execution logic with status code mapping: + +```csharp +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using JdeScoping.Core.ApiContracts.Results; + +namespace JdeScoping.Client.Services; + +/// +/// Base class for API clients with shared HTTP execution logic. +/// +public abstract class ApiClientBase +{ + protected readonly HttpClient HttpClient; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + protected ApiClientBase(HttpClient httpClient) + { + HttpClient = httpClient; + } + + protected async Task> GetAsync(string route, CancellationToken ct = default) + { + return await ExecuteAsync(() => HttpClient.GetAsync(route, ct)); + } + + protected async Task> PostAsync(string route, TBody body, CancellationToken ct = default) + { + return await ExecuteAsync(() => HttpClient.PostAsJsonAsync(route, body, ct)); + } + + protected async Task> PostAsync(string route, CancellationToken ct = default) + { + return await ExecuteAsync(() => HttpClient.PostAsync(route, null, ct)); + } + + protected async Task> GetBytesAsync(string route, CancellationToken ct = default) + { + try + { + var response = await HttpClient.GetAsync(route, ct); + return await MapResponseToBytesAsync(response); + } + catch (Exception ex) + { + return new ApiError(ex.Message); + } + } + + protected async Task> PostMultipartAsync( + string route, + Stream fileStream, + string fileName, + CancellationToken ct = default) + { + try + { + using var content = new MultipartFormDataContent(); + using var streamContent = new StreamContent(fileStream); + content.Add(streamContent, "file", fileName); + + var response = await HttpClient.PostAsync(route, content, ct); + return await MapResponseAsync(response); + } + catch (Exception ex) + { + return new ApiError(ex.Message); + } + } + + private async Task> ExecuteAsync(Func> request) + { + try + { + var response = await request(); + return await MapResponseAsync(response); + } + catch (Exception ex) + { + return new ApiError(ex.Message); + } + } + + private async Task> MapResponseAsync(HttpResponseMessage response) + { + return response.StatusCode switch + { + HttpStatusCode.OK or HttpStatusCode.Created => + await response.Content.ReadFromJsonAsync(JsonOptions) + is T value ? value : new ApiError("Invalid response format"), + + HttpStatusCode.NoContent => + typeof(T) == typeof(Unit) ? (ApiResult)(object)new Unit() : new ApiError("Unexpected empty response"), + + HttpStatusCode.NotFound => new NotFound(), + HttpStatusCode.Unauthorized => new Unauthorized(), + HttpStatusCode.Forbidden => new Forbidden(), + + HttpStatusCode.BadRequest => await ParseValidationErrorAsync(response), + + _ => new ApiError( + await response.Content.ReadAsStringAsync(), + (int)response.StatusCode) + }; + } + + private async Task> MapResponseToBytesAsync(HttpResponseMessage response) + { + return response.StatusCode switch + { + HttpStatusCode.OK => await response.Content.ReadAsByteArrayAsync(), + HttpStatusCode.NotFound => new NotFound(), + HttpStatusCode.Unauthorized => new Unauthorized(), + HttpStatusCode.Forbidden => new Forbidden(), + _ => new ApiError( + await response.Content.ReadAsStringAsync(), + (int)response.StatusCode) + }; + } + + private static async Task> ParseValidationErrorAsync(HttpResponseMessage response) + { + try + { + var content = await response.Content.ReadAsStringAsync(); + var problemDetails = JsonSerializer.Deserialize(content, JsonOptions); + + if (problemDetails?.Errors is { } errors) + { + return new ValidationError(errors); + } + + return new ApiError(content, (int)response.StatusCode); + } + catch + { + return new ApiError("Validation error", (int)response.StatusCode); + } + } + + /// + /// Matches ASP.NET Core ValidationProblemDetails structure. + /// + private sealed class ValidationProblemDetails + { + public Dictionary? Errors { get; set; } + } +} +``` + +### SearchApiClient.cs + +```csharp +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +public class SearchApiClient : ApiClientBase, ISearchApiClient +{ + public SearchApiClient(HttpClient httpClient) : base(httpClient) { } + + public Task>> GetUserSearchesAsync(CancellationToken ct = default) + => GetAsync>(ApiRoutes.Search.Base, ct); + + public Task>> GetQueuedSearchesAsync(CancellationToken ct = default) + => GetAsync>(ApiRoutes.Search.Queue, ct); + + public Task> GetSearchAsync(int id, CancellationToken ct = default) + => GetAsync(ApiRoutes.Search.GetById(id), ct); + + public Task> CopySearchAsync(int id, CancellationToken ct = default) + => GetAsync(ApiRoutes.Search.GetCopy(id), ct); + + public Task> CreateSearchAsync(SearchViewModel search, CancellationToken ct = default) + => PostAsync(ApiRoutes.Search.Base, search, ct); + + public Task> GetResultsAsync(int id, CancellationToken ct = default) + => GetBytesAsync(ApiRoutes.Search.GetResults(id), ct); +} +``` + +### LookupApiClient.cs + +```csharp +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +public class LookupApiClient : ApiClientBase, ILookupApiClient +{ + public LookupApiClient(HttpClient httpClient) : base(httpClient) { } + + public Task>> FindItemsAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindItems(query), ct); + + public Task>> FindProfitCentersAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindProfitCenters(query), ct); + + public Task>> FindWorkCentersAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindWorkCenters(query), ct); + + public Task>> FindOperatorsAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindOperators(query), ct); +} +``` + +## Blazor Component Usage + +```csharp +@page "/searches" +@implements IDisposable +@inject ISearchApiClient SearchApi +@inject NavigationManager NavigationManager + +

My Searches

+ +@if (_loading) +{ +

Loading...

+} +else if (_result.IsSuccess) +{ + +} +else if (_result.IsNotFound) +{ +

No searches found

+} +else if (_result.IsUnauthorized) +{ + // Redirect handled in OnInitializedAsync +} +else if (_result.IsForbidden) +{ +

Access denied

+} +else if (_result.IsValidationError) +{ + +} +else if (_result.IsError) +{ +

@_result.Error.Message

+} + +@code { + private CancellationTokenSource _cts = new(); + private bool _loading = true; + private ApiResult> _result = new ApiError("Not loaded"); + + protected override async Task OnInitializedAsync() + { + _result = await SearchApi.GetUserSearchesAsync(_cts.Token); + _loading = false; + + if (_result.IsUnauthorized) + { + NavigationManager.NavigateTo("/login"); + } + } + + public void Dispose() => _cts.Cancel(); +} +``` + +## DI Registration + +### Client Program.cs + +```csharp +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +``` + +## Migration Path + +1. **Add OneOf package** to Core +2. **Create ApiRoutes.cs** with route constants +3. **Create result types** in `Core/ApiContracts/Results/` +4. **Create client interfaces** (`ISearchApiClient`, etc.) +5. **Create ApiClientBase** with shared HTTP logic +6. **Update controllers** to use `ApiRoutes` constants in attributes +7. **Create client implementations** (`SearchApiClient`, etc.) +8. **Update DI registration** to use new clients +9. **Update Blazor components** to use `ApiResult` pattern +10. **Delete old services** (`SearchService.cs`, `ISearchService.cs`, etc.) + +## Design Notes + +### Why Route Constants Instead of Static Abstract Methods + +Static abstract interface members cannot be used in attribute parameters (attributes require compile-time constants). Using `ApiRoutes` constants allows: +- Controller: `[Route(ApiRoutes.Search.Base)]` +- Client: `GetAsync(ApiRoutes.Search.GetById(id))` + +Both reference the same source of truth. + +### Why Separate Client Interfaces (Not Shared with Controllers) + +Controllers return `ActionResult` for proper HTTP semantics. Clients return `ApiResult` for type-safe error handling. Sharing an interface would require either: +- Controllers returning `ApiResult` (breaks HTTP status codes) +- Complex generic constraints + +Separate interfaces are cleaner and more idiomatic for each context. + +### File Endpoint Considerations + +- Controllers use `IFormFile` for uploads (ASP.NET Core binding) +- Clients use `Stream` (HttpClient multipart) +- `byte[]` for downloads is acceptable for current file sizes +- Future: Consider streaming for very large exports + +### Validation Error Format + +Both sides use ASP.NET Core's `ValidationProblemDetails` format: +```json +{ + "errors": { + "FieldName": ["Error message 1", "Error message 2"] + } +} +``` + +Client parses this into `ValidationError(IReadOnlyDictionary)`. diff --git a/PLANS/2026-01-06-api-client-contracts-implementation.md b/PLANS/2026-01-06-api-client-contracts-implementation.md new file mode 100644 index 0000000..3ceae70 --- /dev/null +++ b/PLANS/2026-01-06-api-client-contracts-implementation.md @@ -0,0 +1,1290 @@ +# API Client Contracts Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Implement shared API contracts in Core that ensure compile-time safety for URLs, parameters, and return types between API and Client projects. + +**Architecture:** Shared `ApiRoutes` constants define URLs usable in controller attributes and client code. Client interfaces (`ISearchApiClient`, etc.) return `ApiResult` using OneOf for type-safe error handling. Controllers continue returning `ActionResult` with proper HTTP semantics. + +**Tech Stack:** .NET 10, OneOf library, Blazor WebAssembly, ASP.NET Core Web API + +--- + +## Task 1: Add OneOf Package to Core + +**Files:** +- Modify: `NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` + +**Step 1: Add OneOf package reference** + +Add to the `` containing PackageReferences in `JdeScoping.Core.csproj`: + +```xml + +``` + +**Step 2: Verify package restores** + +Run: `dotnet restore NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Restore completed successfully + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/JdeScoping.Core.csproj +git commit -m "chore: add OneOf package to Core for API result types" +``` + +--- + +## Task 2: Create Result Types - Error Markers + +**Files:** +- Create: `NEW/src/JdeScoping.Core/ApiContracts/Results/NotFound.cs` +- Create: `NEW/src/JdeScoping.Core/ApiContracts/Results/Unauthorized.cs` +- Create: `NEW/src/JdeScoping.Core/ApiContracts/Results/Forbidden.cs` +- Create: `NEW/src/JdeScoping.Core/ApiContracts/Results/Unit.cs` + +**Step 1: Create NotFound.cs** + +```csharp +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Resource not found (HTTP 404). +/// +public readonly record struct NotFound; +``` + +**Step 2: Create Unauthorized.cs** + +```csharp +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Authentication required (HTTP 401). +/// +public readonly record struct Unauthorized; +``` + +**Step 3: Create Forbidden.cs** + +```csharp +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Access denied (HTTP 403). +/// +public readonly record struct Forbidden; +``` + +**Step 4: Create Unit.cs** + +```csharp +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Empty success type for void operations. +/// +public readonly record struct Unit; +``` + +**Step 5: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 6: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/Results/ +git commit -m "feat: add empty marker result types (NotFound, Unauthorized, Forbidden, Unit)" +``` + +--- + +## Task 3: Create Result Types - Error Types with Data + +**Files:** +- Create: `NEW/src/JdeScoping.Core/ApiContracts/Results/ValidationError.cs` +- Create: `NEW/src/JdeScoping.Core/ApiContracts/Results/ApiError.cs` + +**Step 1: Create ValidationError.cs** + +```csharp +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Validation failed (HTTP 400) with field-level errors. +/// Maps to ASP.NET Core ValidationProblemDetails format. +/// +/// Dictionary mapping field names to error messages. +public readonly record struct ValidationError(IReadOnlyDictionary FieldErrors) +{ + /// + /// Creates a ValidationError from a dictionary of field errors. + /// + public static ValidationError FromDictionary(Dictionary errors) + => new(errors); +} +``` + +**Step 2: Create ApiError.cs** + +```csharp +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// General API error with message and optional status code. +/// +/// Error message describing what went wrong. +/// Optional HTTP status code. +public readonly record struct ApiError(string Message, int? StatusCode = null); +``` + +**Step 3: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 4: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/Results/ +git commit -m "feat: add ValidationError and ApiError result types" +``` + +--- + +## Task 4: Create ApiResult Type + +**Files:** +- Create: `NEW/src/JdeScoping.Core/ApiContracts/Results/ApiResult.cs` + +**Step 1: Create ApiResult.cs** + +```csharp +using OneOf; + +namespace JdeScoping.Core.ApiContracts.Results; + +/// +/// Standard API result type for client-side operations. +/// Represents either success with value T, or one of several error types. +/// +/// The success value type. +[GenerateOneOf] +public partial class ApiResult : OneOfBase +{ + /// Returns true if the result is a success value. + public bool IsSuccess => IsT0; + + /// Returns true if the result is NotFound. + public bool IsNotFound => IsT1; + + /// Returns true if the result is a ValidationError. + public bool IsValidationError => IsT2; + + /// Returns true if the result is Unauthorized. + public bool IsUnauthorized => IsT3; + + /// Returns true if the result is Forbidden. + public bool IsForbidden => IsT4; + + /// Returns true if the result is an ApiError. + public bool IsError => IsT5; + + /// Gets the success value. Throws if not a success. + public T Value => AsT0; + + /// Gets the validation error. Throws if not a validation error. + public ValidationError ValidationError => AsT2; + + /// Gets the API error. Throws if not an API error. + public ApiError Error => AsT5; +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/Results/ApiResult.cs +git commit -m "feat: add ApiResult discriminated union type using OneOf" +``` + +--- + +## Task 5: Create ApiRoutes - Search Routes + +**Files:** +- Create: `NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs` + +**Step 1: Create ApiRoutes.cs with Search routes** + +```csharp +namespace JdeScoping.Core.ApiContracts; + +/// +/// Shared API route constants. Use in controller attributes and client implementations. +/// +public static class ApiRoutes +{ + /// + /// Routes for search API endpoints. + /// + public static class Search + { + /// Base route for search endpoints. + public const string Base = "api/search"; + + /// Route for queued searches. + public const string Queue = "api/search/queue"; + + /// Route template for getting a search by ID (use in controller attributes). + public const string ById = "{id:int}"; + + /// Route template for copying a search (use in controller attributes). + public const string Copy = "{id:int}/copy"; + + /// Route template for getting search results (use in controller attributes). + public const string Results = "{id:int}/results"; + + /// Builds the route to get a specific search. + public static string GetById(int id) => $"api/search/{id}"; + + /// Builds the route to copy a search. + public static string GetCopy(int id) => $"api/search/{id}/copy"; + + /// Builds the route to get search results. + public static string GetResults(int id) => $"api/search/{id}/results"; + } +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs +git commit -m "feat: add ApiRoutes.Search constants and builder methods" +``` + +--- + +## Task 6: Add ApiRoutes - Lookup Routes + +**Files:** +- Modify: `NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs` + +**Step 1: Add Lookup routes to ApiRoutes.cs** + +Add the following class inside the `ApiRoutes` class, after the `Search` class: + +```csharp + /// + /// Routes for lookup/autocomplete API endpoints. + /// + public static class Lookup + { + /// Base route for lookup endpoints. + public const string Base = "api/lookup"; + + /// Route for item lookup. + public const string Items = "api/lookup/items"; + + /// Route for profit center lookup. + public const string ProfitCenters = "api/lookup/profit-centers"; + + /// Route for work center lookup. + public const string WorkCenters = "api/lookup/work-centers"; + + /// Route for operator lookup. + public const string Operators = "api/lookup/operators"; + + /// Builds the route to find items with URL-encoded query. + public static string FindItems(string query) => $"{Items}?q={Uri.EscapeDataString(query)}"; + + /// Builds the route to find profit centers with URL-encoded query. + public static string FindProfitCenters(string query) => $"{ProfitCenters}?q={Uri.EscapeDataString(query)}"; + + /// Builds the route to find work centers with URL-encoded query. + public static string FindWorkCenters(string query) => $"{WorkCenters}?q={Uri.EscapeDataString(query)}"; + + /// Builds the route to find operators with URL-encoded query. + public static string FindOperators(string query) => $"{Operators}?q={Uri.EscapeDataString(query)}"; + } +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs +git commit -m "feat: add ApiRoutes.Lookup constants with URL encoding" +``` + +--- + +## Task 7: Add ApiRoutes - Auth Routes + +**Files:** +- Modify: `NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs` + +**Step 1: Add Auth routes to ApiRoutes.cs** + +Add the following class inside the `ApiRoutes` class: + +```csharp + /// + /// Routes for authentication API endpoints. + /// + public static class Auth + { + /// Base route for auth endpoints. + public const string Base = "api/auth"; + + /// Route to get the public key for credential encryption. + public const string PublicKey = "api/auth/public-key"; + + /// Route for login. + public const string Login = "api/auth/login"; + + /// Route for logout. + public const string Logout = "api/auth/logout"; + + /// Route to get current user info. + public const string Me = "api/auth/me"; + } +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs +git commit -m "feat: add ApiRoutes.Auth constants" +``` + +--- + +## Task 8: Add ApiRoutes - FileIO Routes + +**Files:** +- Modify: `NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs` + +**Step 1: Add FileIO routes to ApiRoutes.cs** + +Add the following class inside the `ApiRoutes` class: + +```csharp + /// + /// Routes for file upload/download API endpoints. + /// + public static class FileIO + { + /// Base route for file IO endpoints. + public const string Base = "api/fileio"; + + /// Route to download work orders template. + public const string DownloadWorkOrders = "api/fileio/workorders/download"; + + /// Route to download items template. + public const string DownloadItems = "api/fileio/items/download"; + + /// Route to download component lots template. + public const string DownloadComponentLots = "api/fileio/componentlots/download"; + + /// Route to download part operations template. + public const string DownloadPartOperations = "api/fileio/partoperations/download"; + + /// Route to upload work orders. + public const string UploadWorkOrders = "api/fileio/workorders/upload"; + + /// Route to upload items. + public const string UploadItems = "api/fileio/items/upload"; + + /// Route to upload component lots. + public const string UploadComponentLots = "api/fileio/componentlots/upload"; + + /// Route to upload part operations. + public const string UploadPartOperations = "api/fileio/partoperations/upload"; + } +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/ApiRoutes.cs +git commit -m "feat: add ApiRoutes.FileIO constants" +``` + +--- + +## Task 9: Create ISearchApiClient Interface + +**Files:** +- Create: `NEW/src/JdeScoping.Core/ApiContracts/ISearchApiClient.cs` + +**Step 1: Create ISearchApiClient.cs** + +```csharp +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for search API operations. +/// +public interface ISearchApiClient +{ + /// Gets all searches for the current user. + Task>> GetUserSearchesAsync(CancellationToken ct = default); + + /// Gets all queued searches. + Task>> GetQueuedSearchesAsync(CancellationToken ct = default); + + /// Gets a specific search by ID. + Task> GetSearchAsync(int id, CancellationToken ct = default); + + /// Copies an existing search to create a new one (returns copy without persisting). + Task> CopySearchAsync(int id, CancellationToken ct = default); + + /// Creates and submits a new search. + Task> CreateSearchAsync(SearchViewModel search, CancellationToken ct = default); + + /// Downloads the results for a completed search as Excel bytes. + Task> GetResultsAsync(int id, CancellationToken ct = default); +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/ISearchApiClient.cs +git commit -m "feat: add ISearchApiClient interface" +``` + +--- + +## Task 10: Create ILookupApiClient Interface + +**Files:** +- Create: `NEW/src/JdeScoping.Core/ApiContracts/ILookupApiClient.cs` + +**Step 1: Create ILookupApiClient.cs** + +```csharp +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for lookup/autocomplete API operations. +/// +public interface ILookupApiClient +{ + /// Finds items matching the search query. + Task>> FindItemsAsync(string query, CancellationToken ct = default); + + /// Finds profit centers matching the search query. + Task>> FindProfitCentersAsync(string query, CancellationToken ct = default); + + /// Finds work centers matching the search query. + Task>> FindWorkCentersAsync(string query, CancellationToken ct = default); + + /// Finds operators (JDE users) matching the search query. + Task>> FindOperatorsAsync(string query, CancellationToken ct = default); +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/ILookupApiClient.cs +git commit -m "feat: add ILookupApiClient interface" +``` + +--- + +## Task 11: Create IAuthApiClient Interface + +**Files:** +- Create: `NEW/src/JdeScoping.Core/ApiContracts/IAuthApiClient.cs` + +**Step 1: Create IAuthApiClient.cs** + +```csharp +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.Models; +using JdeScoping.Core.Models.Auth; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for authentication API operations. +/// +public interface IAuthApiClient +{ + /// Gets the server's RSA public key for encrypting login credentials. + Task> GetPublicKeyAsync(CancellationToken ct = default); + + /// Authenticates with encrypted credentials. + Task> LoginAsync(EncryptedLoginRequest request, CancellationToken ct = default); + + /// Logs out the current user. + Task> LogoutAsync(CancellationToken ct = default); + + /// Gets the current authenticated user's information. + Task> GetCurrentUserAsync(CancellationToken ct = default); +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/IAuthApiClient.cs +git commit -m "feat: add IAuthApiClient interface" +``` + +--- + +## Task 12: Create IFileApiClient Interface + +**Files:** +- Create: `NEW/src/JdeScoping.Core/ApiContracts/IFileApiClient.cs` + +**Step 1: Create IFileApiClient.cs** + +```csharp +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Core.ApiContracts; + +/// +/// Client contract for file upload/download API operations. +/// Note: Uses Stream for client-side; controllers use IFormFile. +/// +public interface IFileApiClient +{ + // Downloads (POST with existing data, returns Excel bytes) + + /// Downloads work orders template, optionally pre-filled with existing data. + Task> DownloadWorkOrdersTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + + /// Downloads items template, optionally pre-filled with existing data. + Task> DownloadItemsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + + /// Downloads component lots template, optionally pre-filled with existing data. + Task> DownloadComponentLotsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + + /// Downloads part operations template, optionally pre-filled with existing data. + Task> DownloadPartOperationsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default); + + // Uploads (multipart form, returns parsed data) + + /// Uploads work orders Excel file and returns parsed data. + Task>> UploadWorkOrdersAsync(Stream fileStream, string fileName, CancellationToken ct = default); + + /// Uploads items Excel file and returns parsed data. + Task>> UploadItemsAsync(Stream fileStream, string fileName, CancellationToken ct = default); + + /// Uploads component lots Excel file and returns parsed data. + Task>> UploadComponentLotsAsync(Stream fileStream, string fileName, CancellationToken ct = default); + + /// Uploads part operations Excel file and returns parsed data. + Task>> UploadPartOperationsAsync(Stream fileStream, string fileName, CancellationToken ct = default); +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Core/JdeScoping.Core.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Core/ApiContracts/IFileApiClient.cs +git commit -m "feat: add IFileApiClient interface" +``` + +--- + +## Task 13: Create ApiClientBase in Client Project + +**Files:** +- Create: `NEW/src/JdeScoping.Client/Services/ApiClientBase.cs` + +**Step 1: Create ApiClientBase.cs** + +```csharp +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using JdeScoping.Core.ApiContracts.Results; + +namespace JdeScoping.Client.Services; + +/// +/// Base class for API clients with shared HTTP execution logic. +/// +public abstract class ApiClientBase +{ + protected readonly HttpClient HttpClient; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + protected ApiClientBase(HttpClient httpClient) + { + HttpClient = httpClient; + } + + protected async Task> GetAsync(string route, CancellationToken ct = default) + { + return await ExecuteAsync(() => HttpClient.GetAsync(route, ct)); + } + + protected async Task> PostAsync(string route, TBody body, CancellationToken ct = default) + { + return await ExecuteAsync(() => HttpClient.PostAsJsonAsync(route, body, ct)); + } + + protected async Task> PostAsync(string route, CancellationToken ct = default) + { + return await ExecuteAsync(() => HttpClient.PostAsync(route, null, ct)); + } + + protected async Task> GetBytesAsync(string route, CancellationToken ct = default) + { + try + { + var response = await HttpClient.GetAsync(route, ct); + return await MapResponseToBytesAsync(response); + } + catch (Exception ex) + { + return new ApiError(ex.Message); + } + } + + protected async Task> PostForBytesAsync(string route, TBody body, CancellationToken ct = default) + { + try + { + var response = await HttpClient.PostAsJsonAsync(route, body, ct); + return await MapResponseToBytesAsync(response); + } + catch (Exception ex) + { + return new ApiError(ex.Message); + } + } + + protected async Task> PostMultipartAsync( + string route, + Stream fileStream, + string fileName, + CancellationToken ct = default) + { + try + { + using var content = new MultipartFormDataContent(); + using var streamContent = new StreamContent(fileStream); + content.Add(streamContent, "file", fileName); + + var response = await HttpClient.PostAsync(route, content, ct); + return await MapResponseAsync(response); + } + catch (Exception ex) + { + return new ApiError(ex.Message); + } + } + + private async Task> ExecuteAsync(Func> request) + { + try + { + var response = await request(); + return await MapResponseAsync(response); + } + catch (Exception ex) + { + return new ApiError(ex.Message); + } + } + + private async Task> MapResponseAsync(HttpResponseMessage response) + { + return response.StatusCode switch + { + HttpStatusCode.OK or HttpStatusCode.Created => + await response.Content.ReadFromJsonAsync(JsonOptions) + is T value ? value : new ApiError("Invalid response format"), + + HttpStatusCode.NoContent => + typeof(T) == typeof(Unit) ? (ApiResult)(object)new Unit() : new ApiError("Unexpected empty response"), + + HttpStatusCode.NotFound => new NotFound(), + HttpStatusCode.Unauthorized => new Unauthorized(), + HttpStatusCode.Forbidden => new Forbidden(), + + HttpStatusCode.BadRequest => await ParseValidationErrorAsync(response), + + _ => new ApiError( + await response.Content.ReadAsStringAsync(), + (int)response.StatusCode) + }; + } + + private async Task> MapResponseToBytesAsync(HttpResponseMessage response) + { + return response.StatusCode switch + { + HttpStatusCode.OK => await response.Content.ReadAsByteArrayAsync(), + HttpStatusCode.NotFound => new NotFound(), + HttpStatusCode.Unauthorized => new Unauthorized(), + HttpStatusCode.Forbidden => new Forbidden(), + _ => new ApiError( + await response.Content.ReadAsStringAsync(), + (int)response.StatusCode) + }; + } + + private static async Task> ParseValidationErrorAsync(HttpResponseMessage response) + { + try + { + var content = await response.Content.ReadAsStringAsync(); + var problemDetails = JsonSerializer.Deserialize(content, JsonOptions); + + if (problemDetails?.Errors is { } errors) + { + return new ValidationError(errors); + } + + return new ApiError(content, (int)response.StatusCode); + } + catch + { + return new ApiError("Validation error", (int)response.StatusCode); + } + } + + /// + /// Matches ASP.NET Core ValidationProblemDetails structure. + /// + private sealed class ValidationProblemDetails + { + public Dictionary? Errors { get; set; } + } +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Client/Services/ApiClientBase.cs +git commit -m "feat: add ApiClientBase with shared HTTP execution logic" +``` + +--- + +## Task 14: Create SearchApiClient Implementation + +**Files:** +- Create: `NEW/src/JdeScoping.Client/Services/SearchApiClient.cs` + +**Step 1: Create SearchApiClient.cs** + +```csharp +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of ISearchApiClient. +/// +public class SearchApiClient : ApiClientBase, ISearchApiClient +{ + public SearchApiClient(HttpClient httpClient) : base(httpClient) { } + + public Task>> GetUserSearchesAsync(CancellationToken ct = default) + => GetAsync>(ApiRoutes.Search.Base, ct); + + public Task>> GetQueuedSearchesAsync(CancellationToken ct = default) + => GetAsync>(ApiRoutes.Search.Queue, ct); + + public Task> GetSearchAsync(int id, CancellationToken ct = default) + => GetAsync(ApiRoutes.Search.GetById(id), ct); + + public Task> CopySearchAsync(int id, CancellationToken ct = default) + => GetAsync(ApiRoutes.Search.GetCopy(id), ct); + + public Task> CreateSearchAsync(SearchViewModel search, CancellationToken ct = default) + => PostAsync(ApiRoutes.Search.Base, search, ct); + + public Task> GetResultsAsync(int id, CancellationToken ct = default) + => GetBytesAsync(ApiRoutes.Search.GetResults(id), ct); +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Client/Services/SearchApiClient.cs +git commit -m "feat: add SearchApiClient implementation" +``` + +--- + +## Task 15: Create LookupApiClient Implementation + +**Files:** +- Create: `NEW/src/JdeScoping.Client/Services/LookupApiClient.cs` + +**Step 1: Create LookupApiClient.cs** + +```csharp +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of ILookupApiClient. +/// +public class LookupApiClient : ApiClientBase, ILookupApiClient +{ + public LookupApiClient(HttpClient httpClient) : base(httpClient) { } + + public Task>> FindItemsAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindItems(query), ct); + + public Task>> FindProfitCentersAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindProfitCenters(query), ct); + + public Task>> FindWorkCentersAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindWorkCenters(query), ct); + + public Task>> FindOperatorsAsync(string query, CancellationToken ct = default) + => GetAsync>(ApiRoutes.Lookup.FindOperators(query), ct); +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Client/Services/LookupApiClient.cs +git commit -m "feat: add LookupApiClient implementation" +``` + +--- + +## Task 16: Create AuthApiClient Implementation + +**Files:** +- Create: `NEW/src/JdeScoping.Client/Services/AuthApiClient.cs` + +**Step 1: Create AuthApiClient.cs** + +```csharp +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.Models; +using JdeScoping.Core.Models.Auth; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of IAuthApiClient. +/// +public class AuthApiClient : ApiClientBase, IAuthApiClient +{ + public AuthApiClient(HttpClient httpClient) : base(httpClient) { } + + public Task> GetPublicKeyAsync(CancellationToken ct = default) + => GetAsync(ApiRoutes.Auth.PublicKey, ct); + + public Task> LoginAsync(EncryptedLoginRequest request, CancellationToken ct = default) + => PostAsync(ApiRoutes.Auth.Login, request, ct); + + public Task> LogoutAsync(CancellationToken ct = default) + => PostAsync(ApiRoutes.Auth.Logout, ct); + + public Task> GetCurrentUserAsync(CancellationToken ct = default) + => GetAsync(ApiRoutes.Auth.Me, ct); +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Client/Services/AuthApiClient.cs +git commit -m "feat: add AuthApiClient implementation" +``` + +--- + +## Task 17: Create FileApiClient Implementation + +**Files:** +- Create: `NEW/src/JdeScoping.Client/Services/FileApiClient.cs` + +**Step 1: Create FileApiClient.cs** + +```csharp +using JdeScoping.Core.ApiContracts; +using JdeScoping.Core.ApiContracts.Results; +using JdeScoping.Core.ViewModels; + +namespace JdeScoping.Client.Services; + +/// +/// HTTP client implementation of IFileApiClient. +/// +public class FileApiClient : ApiClientBase, IFileApiClient +{ + public FileApiClient(HttpClient httpClient) : base(httpClient) { } + + // Downloads + + public Task> DownloadWorkOrdersTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default) + => PostForBytesAsync(ApiRoutes.FileIO.DownloadWorkOrders, existingData, ct); + + public Task> DownloadItemsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default) + => PostForBytesAsync(ApiRoutes.FileIO.DownloadItems, existingData, ct); + + public Task> DownloadComponentLotsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default) + => PostForBytesAsync(ApiRoutes.FileIO.DownloadComponentLots, existingData, ct); + + public Task> DownloadPartOperationsTemplateAsync(IReadOnlyList? existingData = null, CancellationToken ct = default) + => PostForBytesAsync(ApiRoutes.FileIO.DownloadPartOperations, existingData, ct); + + // Uploads + + public Task>> UploadWorkOrdersAsync(Stream fileStream, string fileName, CancellationToken ct = default) + => PostMultipartAsync>(ApiRoutes.FileIO.UploadWorkOrders, fileStream, fileName, ct); + + public Task>> UploadItemsAsync(Stream fileStream, string fileName, CancellationToken ct = default) + => PostMultipartAsync>(ApiRoutes.FileIO.UploadItems, fileStream, fileName, ct); + + public Task>> UploadComponentLotsAsync(Stream fileStream, string fileName, CancellationToken ct = default) + => PostMultipartAsync>(ApiRoutes.FileIO.UploadComponentLots, fileStream, fileName, ct); + + public Task>> UploadPartOperationsAsync(Stream fileStream, string fileName, CancellationToken ct = default) + => PostMultipartAsync>(ApiRoutes.FileIO.UploadPartOperations, fileStream, fileName, ct); +} +``` + +**Step 2: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add NEW/src/JdeScoping.Client/Services/FileApiClient.cs +git commit -m "feat: add FileApiClient implementation" +``` + +--- + +## Task 18: Update SearchController to Use ApiRoutes + +**Files:** +- Modify: `NEW/src/JdeScoping.Api/Controllers/SearchController.cs` + +**Step 1: Add using statement** + +Add at the top of the file: + +```csharp +using JdeScoping.Core.ApiContracts; +``` + +**Step 2: Update Route attribute** + +Change the class-level `[Route]` attribute from: + +```csharp +[Route("api/search")] +``` + +To: + +```csharp +[Route(ApiRoutes.Search.Base)] +``` + +**Step 3: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Api/JdeScoping.Api.csproj` +Expected: Build succeeded + +**Step 4: Commit** + +```bash +git add NEW/src/JdeScoping.Api/Controllers/SearchController.cs +git commit -m "refactor: update SearchController to use ApiRoutes constants" +``` + +--- + +## Task 19: Update LookupController to Use ApiRoutes + +**Files:** +- Modify: `NEW/src/JdeScoping.Api/Controllers/LookupController.cs` + +**Step 1: Add using statement** + +Add at the top of the file: + +```csharp +using JdeScoping.Core.ApiContracts; +``` + +**Step 2: Update Route attribute** + +Change the class-level `[Route]` attribute from: + +```csharp +[Route("api/lookup")] +``` + +To: + +```csharp +[Route(ApiRoutes.Lookup.Base)] +``` + +**Step 3: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Api/JdeScoping.Api.csproj` +Expected: Build succeeded + +**Step 4: Commit** + +```bash +git add NEW/src/JdeScoping.Api/Controllers/LookupController.cs +git commit -m "refactor: update LookupController to use ApiRoutes constants" +``` + +--- + +## Task 20: Update AuthController to Use ApiRoutes + +**Files:** +- Modify: `NEW/src/JdeScoping.Api/Controllers/AuthController.cs` + +**Step 1: Add using statement** + +Add at the top of the file: + +```csharp +using JdeScoping.Core.ApiContracts; +``` + +**Step 2: Update Route attribute** + +Change the class-level `[Route]` attribute from: + +```csharp +[Route("api/auth")] +``` + +To: + +```csharp +[Route(ApiRoutes.Auth.Base)] +``` + +**Step 3: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Api/JdeScoping.Api.csproj` +Expected: Build succeeded + +**Step 4: Commit** + +```bash +git add NEW/src/JdeScoping.Api/Controllers/AuthController.cs +git commit -m "refactor: update AuthController to use ApiRoutes constants" +``` + +--- + +## Task 21: Update FileIOController to Use ApiRoutes + +**Files:** +- Modify: `NEW/src/JdeScoping.Api/Controllers/FileController.cs` + +**Step 1: Add using statement** + +Add at the top of the file: + +```csharp +using JdeScoping.Core.ApiContracts; +``` + +**Step 2: Update Route attribute** + +Change the class-level `[Route]` attribute from: + +```csharp +[Route("api/fileio")] +``` + +To: + +```csharp +[Route(ApiRoutes.FileIO.Base)] +``` + +**Step 3: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Api/JdeScoping.Api.csproj` +Expected: Build succeeded + +**Step 4: Commit** + +```bash +git add NEW/src/JdeScoping.Api/Controllers/FileController.cs +git commit -m "refactor: update FileIOController to use ApiRoutes constants" +``` + +--- + +## Task 22: Register New API Clients in Client DI + +**Files:** +- Modify: `NEW/src/JdeScoping.Client/Program.cs` + +**Step 1: Add using statement** + +Add at the top of the file: + +```csharp +using JdeScoping.Core.ApiContracts; +``` + +**Step 2: Add new client registrations** + +Find where services are registered and add: + +```csharp +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +``` + +**Step 3: Verify build** + +Run: `dotnet build NEW/src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 4: Commit** + +```bash +git add NEW/src/JdeScoping.Client/Program.cs +git commit -m "feat: register new API clients in Client DI container" +``` + +--- + +## Task 23: Full Solution Build Verification + +**Files:** +- None (verification only) + +**Step 1: Build entire solution** + +Run: `dotnet build NEW/JdeScopingTool.sln` +Expected: Build succeeded with no errors + +**Step 2: Run existing tests** + +Run: `dotnet test NEW/JdeScopingTool.sln --no-build` +Expected: All tests pass + +--- + +## Summary + +This implementation plan adds: + +1. **Core/ApiContracts/Results/** - Result types (ApiResult, NotFound, ValidationError, etc.) +2. **Core/ApiContracts/ApiRoutes.cs** - Shared route constants +3. **Core/ApiContracts/I*ApiClient.cs** - Client interface contracts +4. **Client/Services/ApiClientBase.cs** - Shared HTTP execution logic +5. **Client/Services/*ApiClient.cs** - Client implementations +6. **Controller updates** - Use ApiRoutes constants + +The old services (`SearchService`, `LookupService`, `AuthService`, `FileService`) remain functional. Migration of Blazor components to use the new clients is a separate follow-up task. diff --git a/PLANS/2026-01-06-api-client-tests-design.md b/PLANS/2026-01-06-api-client-tests-design.md new file mode 100644 index 0000000..10507f1 --- /dev/null +++ b/PLANS/2026-01-06-api-client-tests-design.md @@ -0,0 +1,260 @@ +# API Client Tests Design + +## Purpose + +Update unit tests for Api, Api.Integration, and Client projects to use the new API contracts (`ApiRoutes`, `ApiResult`, `I*ApiClient` interfaces). + +## Goals + +1. Replace hardcoded route strings with `ApiRoutes.*` constants +2. Add comprehensive unit tests for API client base behavior +3. Add focused unit tests for each API client (success + representative error) +4. Add integration tests using actual `*ApiClient` classes against test server +5. Cover edge cases: malformed payloads, empty responses, network errors + +## Test Structure + +``` +tests/ +├── JdeScoping.Client.Tests/ +│ └── Services/ +│ ├── CryptoServiceTests.cs (existing) +│ ├── ApiClientBaseTests.cs (new - ALL 6 ApiResult cases + edge cases) +│ ├── SearchApiClientTests.cs (new - success + 1 error per method) +│ ├── LookupApiClientTests.cs (new - success + 1 error per method) +│ ├── AuthApiClientTests.cs (new - success + 1 error per method) +│ └── FileApiClientTests.cs (new - success + 1 error per method) +│ +├── JdeScoping.Api.IntegrationTests/ +│ ├── AuthenticationTests.cs (update routes to ApiRoutes.*) +│ ├── FileControllerIntegrationTests.cs (update routes to ApiRoutes.*) +│ ├── SignalRTests.cs (update routes if needed) +│ └── ClientIntegrationTests/ (new folder) +│ ├── SearchApiClientIntegrationTests.cs +│ ├── LookupApiClientIntegrationTests.cs +│ ├── AuthApiClientIntegrationTests.cs +│ └── FileApiClientIntegrationTests.cs +``` + +## Unit Test Approach + +### ApiClientBaseTests - Full Coverage + +Tests ALL 6 HTTP status code → ApiResult mappings plus edge cases: + +```csharp +public class ApiClientBaseTests +{ + // Core status code mappings (test once here, not repeated per client) + [Fact] Task GetAsync_Returns200_MapsToSuccessValue() + [Fact] Task GetAsync_Returns404_MapsToNotFound() + [Fact] Task GetAsync_Returns400_MapsToValidationError() + [Fact] Task GetAsync_Returns401_MapsToUnauthorized() + [Fact] Task GetAsync_Returns403_MapsToForbidden() + [Fact] Task GetAsync_Returns500_MapsToApiError() + + // Edge cases (malformed responses) + [Fact] Task GetAsync_Returns200_EmptyBody_MapsToApiError() + [Fact] Task GetAsync_Returns200_InvalidJson_MapsToApiError() + [Fact] Task GetAsync_Returns204_MapsToApiError() + [Fact] Task GetAsync_Returns422_MapsToValidationError() + [Fact] Task GetAsync_NetworkException_MapsToApiError() + [Fact] Task GetAsync_Timeout_MapsToApiError() + + // Also test PostAsync, GetBytesAsync, PostMultipartAsync +} +``` + +### Client Test Pattern - Lean Coverage + +Each client tests: +1. Correct route called (verify path AND HTTP method) +2. Query string encoding (for lookup methods) +3. Success case with response deserialization +4. One representative error case (usually 401 for auth-required, 404 for not found) + +```csharp +public class SearchApiClientTests +{ + private readonly MockHttpMessageHandler _mockHttp; + private readonly SearchApiClient _client; + + // Route + method verification + [Fact] Task GetUserSearchesAsync_CallsCorrectRoute_WithGetMethod() + [Fact] Task CreateSearchAsync_CallsCorrectRoute_WithPostMethod() + + // Success cases + [Fact] Task GetUserSearchesAsync_Success_ReturnsSearchList() + [Fact] Task GetSearchAsync_Success_ReturnsSearch() + [Fact] Task CreateSearchAsync_Success_ReturnsId() + + // Representative error (not all 6 - those are tested in ApiClientBaseTests) + [Fact] Task GetSearchAsync_404_ReturnsNotFound() + [Fact] Task GetUserSearchesAsync_401_ReturnsUnauthorized() +} + +public class LookupApiClientTests +{ + // Query string encoding is important for lookup methods + [Fact] Task FindItemsAsync_EncodesQueryString_Correctly() + [Fact] Task FindItemsAsync_WithSpecialChars_EncodesCorrectly() + + [Fact] Task FindItemsAsync_Success_ReturnsItemList() + [Fact] Task FindOperatorsAsync_Success_ReturnsUserList() +} +``` + +## Integration Test Approach + +### Shared HttpClient for Auth State + +**Critical:** Use a single `HttpClient` instance with `HandleCookies = true` for all authenticated calls. Auth is cookie-based. + +```csharp +public class ClientIntegrationTestBase : IClassFixture +{ + protected readonly TestWebApplicationFactory Factory; + protected readonly HttpClient SharedClient; // Single client, preserves cookies + + // API clients share the authenticated HttpClient + protected readonly ISearchApiClient SearchClient; + protected readonly ILookupApiClient LookupClient; + protected readonly IAuthApiClient AuthClient; + protected readonly IFileApiClient FileClient; + + public ClientIntegrationTestBase(TestWebApplicationFactory factory) + { + Factory = factory; + SharedClient = factory.CreateClient(new WebApplicationFactoryClientOptions + { + HandleCookies = true, + AllowAutoRedirect = false + }); + + // All clients share the same HttpClient (cookie container) + SearchClient = new SearchApiClient(SharedClient); + LookupClient = new LookupApiClient(SharedClient); + FileClient = new FileApiClient(SharedClient); + + // AuthApiClient needs crypto service + var cryptoService = CreateCryptoService(SharedClient); + AuthClient = new AuthApiClient(SharedClient, cryptoService); + } + + protected async Task LoginAsync(string username = "testuser", string password = "testpass") + { + var result = await AuthClient.LoginAsync(new EncryptedLoginRequest(...)); + result.IsSuccess.ShouldBeTrue(); + } + + // For testing unauthorized scenarios, create a fresh client + protected HttpClient CreateFreshClient() => Factory.CreateClient(new WebApplicationFactoryClientOptions + { + HandleCookies = false, // No cookies = no auth + AllowAutoRedirect = false + }); +} +``` + +### Integration Test Pattern + +```csharp +public class SearchApiClientIntegrationTests : ClientIntegrationTestBase +{ + public SearchApiClientIntegrationTests(TestWebApplicationFactory factory) : base(factory) { } + + [Fact] + public async Task GetUserSearchesAsync_WithAuth_ReturnsSearchList() + { + // Arrange + await LoginAsync(); + + // Act + var result = await SearchClient.GetUserSearchesAsync(); + + // Assert + result.IsSuccess.ShouldBeTrue(); + result.Value.ShouldNotBeNull(); + } + + [Fact] + public async Task GetUserSearchesAsync_WithoutAuth_ReturnsUnauthorized() + { + // Use fresh client without cookies + var freshClient = new SearchApiClient(CreateFreshClient()); + + var result = await freshClient.GetUserSearchesAsync(); + + result.IsUnauthorized.ShouldBeTrue(); + } +} +``` + +### Test Data and Cleanup + +Integration tests use: +- `UseFakeAuth = true` in test server (no real LDAP) +- In-memory or test database with seeded data +- xUnit collection fixtures to avoid parallelization issues with shared state + +```csharp +[Collection("IntegrationTests")] // Prevents parallel execution +public class SearchApiClientIntegrationTests : ClientIntegrationTestBase +{ + // Tests run sequentially within collection +} +``` + +### Update Existing Tests + +Replace hardcoded routes: + +```csharp +// Before: +var response = await _client.GetAsync("/api/search"); + +// After: +var response = await _client.GetAsync(ApiRoutes.Search.Base); +``` + +## Test Cases Per Client (Revised - Lean) + +### ApiClientBaseTests (~18 tests) +- 6 status code mappings × 3 HTTP methods (GET, POST, multipart) +- 6 edge case tests (empty body, invalid JSON, 204, 422, network error, timeout) + +### SearchApiClient (~12 tests) +- 6 route/method verifications +- 6 success cases (one per method) +- 2 representative error cases + +### LookupApiClient (~10 tests) +- 4 route/method verifications +- 2 query string encoding tests +- 4 success cases + +### AuthApiClient (~8 tests) +- 4 route/method verifications +- 4 success cases + +### FileApiClient (~16 tests) +- 8 route/method verifications +- 8 success cases + +### Integration Tests (~12 tests) +- 3 per client (with auth, without auth, specific scenario) + +## Total Test Count (Revised) + +- Unit tests: ~64 (down from 132) +- Integration tests: ~12 +- Total: ~76 new tests + +## Key Design Decisions + +1. **All 6 ApiResult cases tested in ApiClientBaseTests only** - Not repeated per client method +2. **Shared HttpClient for auth** - Single client with cookies for authenticated integration tests +3. **Fresh client for unauthorized tests** - New HttpClient without cookies +4. **Query string encoding** - Explicitly tested for lookup methods +5. **Edge cases covered** - Malformed JSON, empty bodies, network errors +6. **No Activator.CreateInstance** - Direct instantiation with shared HttpClient diff --git a/PLANS/2026-01-06-blazor-migration-design.md b/PLANS/2026-01-06-blazor-migration-design.md new file mode 100644 index 0000000..2e42fbf --- /dev/null +++ b/PLANS/2026-01-06-blazor-migration-design.md @@ -0,0 +1,227 @@ +# Blazor Component Migration Design + +## Purpose + +Migrate all Blazor components from using the old `I*Service` interfaces to the new `I*ApiClient` interfaces, implementing proper error handling with the `ApiResult` discriminated union pattern. + +## Architecture + +### View Model Mapping + +The client has its own view models (`JdeScoping.Client.Models.*`) that differ from Core view models (`JdeScoping.Core.ViewModels.*`): + +| Client | Core | Differences | +|--------|------|-------------| +| `SearchViewModel` | `SearchViewModel` | Client uses `string Status`, Core uses `SearchStatus` enum | +| `SearchCriteriaViewModel` | `SearchCriteria` | Same structure, different namespaces | +| `ItemViewModel` | `ItemViewModel` | Same | +| `OperatorViewModel` | `JdeUserViewModel` | Different names | + +**Strategy**: Create mapping extension methods to convert Core -> Client view models. + +### Error Handling Strategy + +**Hybrid approach:** +- **Global 401 handling**: `AuthRedirectHandler` (DelegatingHandler) intercepts all 401 responses and redirects to `/login` +- **Component-level handling**: Components use `result.Switch()` for all other cases (success, not found, validation errors, general errors) + +### Why Keep Unauthorized in ApiResult + +Even with global 401 handling, we keep `Unauthorized` in the type for: +1. **Completeness** - Type accurately represents all possible server responses +2. **Defense in depth** - Fallback if handler fails or is bypassed +3. **Testing** - Can test unauthorized scenarios without HTTP layer +4. **Future flexibility** - May need component-specific handling later + +### Pattern Usage + +Components use `result.Switch()` directly - no shared helper method. This provides: +- Explicit handling at each call site +- Flexibility for different UI patterns per component +- Clear, readable code without indirection + +## Components + +### AuthRedirectHandler + +```csharp +public class AuthRedirectHandler : DelegatingHandler +{ + private readonly NavigationManager _navigationManager; + + public AuthRedirectHandler(NavigationManager navigationManager) + { + _navigationManager = navigationManager; + } + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = await base.SendAsync(request, cancellationToken); + + if (response.StatusCode == HttpStatusCode.Unauthorized) + { + var returnUrl = Uri.EscapeDataString(_navigationManager.Uri); + _navigationManager.NavigateTo($"/login?returnUrl={returnUrl}", forceLoad: true); + } + + return response; + } +} +``` + +### View Model Mapping Extensions + +```csharp +public static class ViewModelMappingExtensions +{ + public static ClientSearchViewModel ToClient(this Core.ViewModels.SearchViewModel vm) => new() + { + Id = vm.Id, + Name = vm.Name, + UserName = vm.UserName, + Status = vm.Status.ToString(), + SubmitDt = vm.SubmitDt, + StartDt = vm.StartDt, + EndDt = vm.EndDt, + Criteria = vm.Criteria?.ToClientCriteria() ?? new() + }; + + public static Core.ViewModels.SearchViewModel ToCore(this ClientSearchViewModel vm) => new() + { + Id = vm.Id, + Name = vm.Name, + UserName = vm.UserName, + Status = Enum.Parse(vm.Status), + SubmitDt = vm.SubmitDt, + StartDt = vm.StartDt, + EndDt = vm.EndDt, + Criteria = vm.Criteria.ToCoreCriteria() + }; +} +``` + +### Component Pattern + +Before: +```csharp +@inject ISearchService SearchService + +var searches = await SearchService.GetUserSearchesAsync(); +``` + +After: +```csharp +@inject ISearchApiClient SearchApi + +var result = await SearchApi.GetUserSearchesAsync(); +result.Switch( + searches => { _searches = searches.Select(s => s.ToClient()).ToList(); }, + notFound => { _errorMessage = "Not found"; }, + validation => { _errorMessage = validation.Message; }, + unauthorized => { /* handled globally, but fallback */ }, + forbidden => { _errorMessage = "Access denied"; }, + error => { _errorMessage = error.Message; } +); +``` + +## Files to Change + +### Create +- `Client/Http/AuthRedirectHandler.cs` - Global 401 redirect handler +- `Client/Extensions/ViewModelMappingExtensions.cs` - Core <-> Client mapping + +### Modify (12 files) + +**Pages:** +1. `Pages/Searches.razor` - Uses `ISearchService` +2. `Pages/SearchEdit.razor` - Uses `ISearchService`, `IFileService` +3. `Pages/SearchQueue.razor` - Uses `ISearchService` +4. `Pages/Login.razor` - Uses `IAuthService` +5. `Layout/MainLayout.razor` - Uses `IAuthService` + +**Filter Panels:** +6. `Components/FilterPanels/ItemNumberFilterPanel.razor` - Uses `ILookupService`, `IFileService` +7. `Components/FilterPanels/WorkCenterFilterPanel.razor` - Uses `ILookupService` +8. `Components/FilterPanels/ProfitCenterFilterPanel.razor` - Uses `ILookupService` +9. `Components/FilterPanels/OperatorFilterPanel.razor` - Uses `ILookupService` +10. `Components/FilterPanels/WorkOrderFilterPanel.razor` - Uses `IFileService` +11. `Components/FilterPanels/ComponentLotFilterPanel.razor` - Uses `IFileService` +12. `Components/FilterPanels/PartOperationFilterPanel.razor` - Uses `IFileService` + +### Update +- `Client/Program.cs` - Register AuthRedirectHandler, configure HttpClient, remove old services + +### Delete (8 old service files) +- `Client/Services/ISearchService.cs` +- `Client/Services/SearchService.cs` +- `Client/Services/ILookupService.cs` +- `Client/Services/LookupService.cs` +- `Client/Services/IAuthService.cs` +- `Client/Services/AuthService.cs` +- `Client/Services/IFileService.cs` +- `Client/Services/FileService.cs` + +### Keep (not migrating) +- `IRefreshStatusService` / `RefreshStatusService` - No corresponding API client yet +- `IHubConnectionService` / `HubConnectionService` - SignalR, not HTTP +- `ICryptoService` / `CryptoService` - Client-side encryption + +## DI Registration + +```csharp +// Add handler +builder.Services.AddTransient(); + +// Configure HttpClient with handler pipeline +builder.Services.AddScoped(sp => +{ + var navigationManager = sp.GetRequiredService(); + var handler = new AuthRedirectHandler(navigationManager) + { + InnerHandler = new HttpClientHandler() + }; + return new HttpClient(handler) + { + BaseAddress = new Uri(sp.GetRequiredService().BaseAddress) + }; +}); + +// Remove old service registrations +// - builder.Services.AddScoped(); +// - builder.Services.AddScoped(); +// - builder.Services.AddScoped(); +// - builder.Services.AddScoped(); +``` + +## Migration Order + +1. Create foundation files (AuthRedirectHandler, ViewModelMappingExtensions) +2. Update Program.cs to register handler +3. Migrate pages one at a time, testing each: + - Searches.razor (simplest, read-only) + - SearchQueue.razor (read-only) + - SearchEdit.razor (most complex, read + write) + - Login.razor (auth) + - MainLayout.razor (logout) +4. Migrate filter panels: + - ItemNumberFilterPanel (lookup + file) + - WorkCenterFilterPanel (lookup only) + - ProfitCenterFilterPanel (lookup only) + - OperatorFilterPanel (lookup only) + - WorkOrderFilterPanel (file only) + - ComponentLotFilterPanel (file only) + - PartOperationFilterPanel (file only) +5. Delete old service files after all components migrated +6. Final verification pass + +## Acceptance Criteria + +- [ ] All 401 responses redirect to `/login` with return URL +- [ ] Components display appropriate error messages for each error type +- [ ] No old `I*Service` interfaces remain in use +- [ ] Old service files deleted +- [ ] All components compile and function correctly +- [ ] Authentication flow works end-to-end +- [ ] View model mapping works correctly (Core <-> Client) diff --git a/PLANS/2026-01-06-blazor-migration-implementation.md b/PLANS/2026-01-06-blazor-migration-implementation.md new file mode 100644 index 0000000..c4e2823 --- /dev/null +++ b/PLANS/2026-01-06-blazor-migration-implementation.md @@ -0,0 +1,1107 @@ +# Blazor Component Migration Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Migrate all Blazor components from old `I*Service` interfaces to new `I*ApiClient` interfaces with proper `ApiResult` error handling. + +**Architecture:** Create AuthRedirectHandler for global 401 handling, view model mapping extensions for Core<->Client conversion, then update each component to inject API clients and use `result.Switch()` pattern. + +**Tech Stack:** Blazor WebAssembly, OneOf discriminated unions, DelegatingHandler, extension methods + +--- + +### Task 1: Create AuthRedirectHandler + +**Files:** +- Create: `src/JdeScoping.Client/Http/AuthRedirectHandler.cs` + +**Step 1: Create the handler file** + +```csharp +using System.Net; +using Microsoft.AspNetCore.Components; + +namespace JdeScoping.Client.Http; + +/// +/// HTTP message handler that intercepts 401 Unauthorized responses +/// and redirects to the login page with return URL. +/// +public class AuthRedirectHandler : DelegatingHandler +{ + private readonly NavigationManager _navigationManager; + + public AuthRedirectHandler(NavigationManager navigationManager) + { + _navigationManager = navigationManager; + } + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = await base.SendAsync(request, cancellationToken); + + if (response.StatusCode == HttpStatusCode.Unauthorized) + { + var returnUrl = Uri.EscapeDataString(_navigationManager.Uri); + _navigationManager.NavigateTo($"/login?returnUrl={returnUrl}", forceLoad: true); + } + + return response; + } +} +``` + +**Step 2: Verify file compiles** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add src/JdeScoping.Client/Http/AuthRedirectHandler.cs +git commit -m "feat(client): add AuthRedirectHandler for global 401 redirect" +``` + +--- + +### Task 2: Create View Model Mapping Extensions + +**Files:** +- Create: `src/JdeScoping.Client/Extensions/ViewModelMappingExtensions.cs` + +**Step 1: Create the mapping extensions file** + +```csharp +using JdeScoping.Client.Models; +using JdeScoping.Core.Models.Enums; +using JdeScoping.Core.Models.Search; +using CoreSearch = JdeScoping.Core.ViewModels.SearchViewModel; +using CoreItem = JdeScoping.Core.ViewModels.ItemViewModel; +using CoreWorkOrder = JdeScoping.Core.ViewModels.WorkOrderViewModel; +using CoreProfitCenter = JdeScoping.Core.ViewModels.ProfitCenterViewModel; +using CoreWorkCenter = JdeScoping.Core.ViewModels.WorkCenterViewModel; +using CoreLot = JdeScoping.Core.ViewModels.LotViewModel; +using CorePartOp = JdeScoping.Core.ViewModels.PartOperationViewModel; +using CoreJdeUser = JdeScoping.Core.ViewModels.JdeUserViewModel; + +namespace JdeScoping.Client.Extensions; + +/// +/// Extension methods for mapping between Core and Client view models. +/// +public static class ViewModelMappingExtensions +{ + // SearchViewModel: Core -> Client + public static SearchViewModel ToClient(this CoreSearch vm) => new() + { + Id = vm.Id, + Name = vm.Name, + UserName = vm.UserName, + Status = vm.Status.ToString(), + SubmitDt = vm.SubmitDt, + StartDt = vm.StartDt, + EndDt = vm.EndDt, + Criteria = vm.Criteria?.ToClientCriteria() ?? new() + }; + + // SearchViewModel: Client -> Core + public static CoreSearch ToCore(this SearchViewModel vm) => new() + { + Id = vm.Id, + Name = vm.Name, + UserName = vm.UserName, + Status = Enum.TryParse(vm.Status, out var status) ? status : SearchStatus.New, + SubmitDt = vm.SubmitDt, + StartDt = vm.StartDt, + EndDt = vm.EndDt, + Criteria = vm.Criteria.ToCoreCriteria() + }; + + // SearchCriteria: Core -> Client + public static SearchCriteriaViewModel ToClientCriteria(this SearchCriteria criteria) + { + var client = new SearchCriteriaViewModel + { + MinimumDt = criteria.MinimumDt, + MaximumDt = criteria.MaximumDt, + ExtractMisData = criteria.ExtractMisData + }; + + // Map work orders (Core has just numbers, Client has full objects) + client.WorkOrders = criteria.WorkOrderNumbers + .Select(n => new CoreWorkOrder { WorkOrderNumber = n }) + .ToList(); + + // Map items + client.Items = criteria.ItemNumbers + .Select(n => new CoreItem { ItemNumber = n }) + .ToList(); + + // Map profit centers + client.ProfitCenters = criteria.ProfitCenters + .Select(pc => new CoreProfitCenter { ProfitCenterCode = pc }) + .ToList(); + + // Map work centers + client.WorkCenters = criteria.WorkCenters + .Select(wc => new CoreWorkCenter { WorkCenterCode = wc }) + .ToList(); + + // Map operators + client.Operators = criteria.OperatorIDs + .Select(id => new OperatorViewModel { UserId = id }) + .ToList(); + + // Map component lots (Core and Client both use LotViewModel) + client.ComponentLots = criteria.ComponentLotNumbers + .Select(l => new ComponentLotViewModel { LotNumber = l.LotNumber, ItemNumber = l.ItemNumber }) + .ToList(); + + // Map part operations (same structure) + client.PartOperations = criteria.PartOperations.ToList(); + + return client; + } + + // SearchCriteria: Client -> Core + public static SearchCriteria ToCoreCriteria(this SearchCriteriaViewModel criteria) => new() + { + MinimumDt = criteria.MinimumDt, + MaximumDt = criteria.MaximumDt, + ExtractMisData = criteria.ExtractMisData, + WorkOrderNumbers = criteria.WorkOrders.Select(wo => wo.WorkOrderNumber).ToList(), + ItemNumbers = criteria.Items.Select(i => i.ItemNumber).ToList(), + ProfitCenters = criteria.ProfitCenters.Select(pc => pc.ProfitCenterCode).ToList(), + WorkCenters = criteria.WorkCenters.Select(wc => wc.WorkCenterCode).ToList(), + OperatorIDs = criteria.Operators.Select(o => o.UserId).ToList(), + ComponentLotNumbers = criteria.ComponentLots + .Select(l => new CoreLot { LotNumber = l.LotNumber, ItemNumber = l.ItemNumber }) + .ToList(), + PartOperations = criteria.PartOperations.ToList() + }; + + // JdeUserViewModel -> OperatorViewModel + public static OperatorViewModel ToClientOperator(this CoreJdeUser vm) => new() + { + AddressNumber = (int)vm.AddressNumber, + UserId = vm.UserId, + FullName = vm.FullName + }; + + // Collection helpers + public static List ToClientList(this IEnumerable list) => + list.Select(s => s.ToClient()).ToList(); + + public static List ToClientOperatorList(this IEnumerable list) => + list.Select(u => u.ToClientOperator()).ToList(); +} +``` + +**Step 2: Verify file compiles** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add src/JdeScoping.Client/Extensions/ViewModelMappingExtensions.cs +git commit -m "feat(client): add ViewModelMappingExtensions for Core<->Client mapping" +``` + +--- + +### Task 3: Update Program.cs with HttpClient Handler Pipeline + +**Files:** +- Modify: `src/JdeScoping.Client/Program.cs` + +**Step 1: Update HttpClient registration to use handler** + +Replace the current HttpClient registration with handler pipeline: + +```csharp +// OLD: +builder.Services.AddScoped(sp => new HttpClient +{ + BaseAddress = new Uri(builder.HostEnvironment.BaseAddress) +}); + +// NEW: +builder.Services.AddScoped(); +builder.Services.AddScoped(sp => +{ + var navigationManager = sp.GetRequiredService(); + var handler = new AuthRedirectHandler(navigationManager) + { + InnerHandler = new HttpClientHandler() + }; + return new HttpClient(handler) + { + BaseAddress = new Uri(builder.HostEnvironment.BaseAddress) + }; +}); +``` + +**Step 2: Add required using statement** + +Add at top of file: +```csharp +using JdeScoping.Client.Http; +``` + +**Step 3: Verify build** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 4: Commit** + +```bash +git add src/JdeScoping.Client/Program.cs +git commit -m "feat(client): configure HttpClient with AuthRedirectHandler" +``` + +--- + +### Task 4: Migrate Searches.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Pages/Searches.razor` + +**Step 1: Update inject statement** + +Change: +```razor +@inject ISearchService SearchService +``` +To: +```razor +@inject ISearchApiClient SearchApi +``` + +**Step 2: Add using for extensions** + +Add after @page directives: +```razor +@using JdeScoping.Client.Extensions +``` + +**Step 3: Update LoadSearchesAsync method** + +Replace the entire method with: + +```csharp +private async Task LoadSearchesAsync() +{ + _isLoading = true; + _errorMessage = null; + try + { + var result = await SearchApi.GetUserSearchesAsync(); + result.Switch( + searches => { _searches = searches.ToClientList(); }, + notFound => { _errorMessage = "No searches found."; _searches = []; }, + validation => { _errorMessage = validation.Message; }, + unauthorized => { _errorMessage = "Session expired. Please login again."; }, + forbidden => { _errorMessage = "Access denied."; }, + error => { _errorMessage = error.Message; } + ); + } + finally + { + _isLoading = false; + } +} +``` + +**Step 4: Add error message field and display** + +Add field in @code section: +```csharp +private string? _errorMessage; +``` + +Add error display in markup after loading indicator: +```razor +@if (!string.IsNullOrEmpty(_errorMessage)) +{ + + @_errorMessage + +} +``` + +**Step 5: Verify build** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 6: Commit** + +```bash +git add src/JdeScoping.Client/Pages/Searches.razor +git commit -m "feat(client): migrate Searches.razor to ISearchApiClient" +``` + +--- + +### Task 5: Migrate SearchQueue.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Pages/SearchQueue.razor` + +**Step 1: Update inject statement** + +Change: +```razor +@inject ISearchService SearchService +``` +To: +```razor +@inject ISearchApiClient SearchApi +``` + +**Step 2: Add using for extensions** + +Add after @page directive: +```razor +@using JdeScoping.Client.Extensions +``` + +**Step 3: Update LoadQueueAsync method** + +Replace with: + +```csharp +private async Task LoadQueueAsync() +{ + _isLoading = true; + _errorMessage = null; + try + { + var result = await SearchApi.GetQueuedSearchesAsync(); + result.Switch( + searches => { _searches = searches.ToClientList(); }, + notFound => { _errorMessage = "Queue not found."; _searches = []; }, + validation => { _errorMessage = validation.Message; }, + unauthorized => { _errorMessage = "Session expired. Please login again."; }, + forbidden => { _errorMessage = "Access denied."; }, + error => { _errorMessage = error.Message; } + ); + } + finally + { + _isLoading = false; + } +} +``` + +**Step 4: Add error message field and display** + +Add field: +```csharp +private string? _errorMessage; +``` + +Add error display after loading indicator. + +**Step 5: Verify build and commit** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` + +```bash +git add src/JdeScoping.Client/Pages/SearchQueue.razor +git commit -m "feat(client): migrate SearchQueue.razor to ISearchApiClient" +``` + +--- + +### Task 6: Migrate SearchEdit.razor - Part 1 (Load Methods) + +**Files:** +- Modify: `src/JdeScoping.Client/Pages/SearchEdit.razor` + +**Step 1: Update inject statements** + +Change: +```razor +@inject ISearchService SearchService +@inject IFileService FileService +``` +To: +```razor +@inject ISearchApiClient SearchApi +@inject IFileApiClient FileApi +``` + +**Step 2: Add using for extensions** + +```razor +@using JdeScoping.Client.Extensions +``` + +**Step 3: Update LoadSearchAsync method** + +Replace entire method with: + +```csharp +private async Task LoadSearchAsync() +{ + _isLoading = true; + _errorMessage = null; + try + { + if (CopySearchId.HasValue) + { + var result = await SearchApi.CopySearchAsync(CopySearchId.Value); + result.Switch( + copied => { + _search = copied.ToClient(); + _search.Id = 0; + _search.Status = "New"; + }, + notFound => { _errorMessage = "Search to copy not found."; }, + validation => { _errorMessage = validation.Message; }, + unauthorized => { _errorMessage = "Session expired."; }, + forbidden => { _errorMessage = "Access denied."; }, + error => { _errorMessage = error.Message; } + ); + } + else if (Id.HasValue && Id.Value > 0) + { + var result = await SearchApi.GetSearchAsync(Id.Value); + result.Switch( + loaded => { _search = loaded.ToClient(); }, + notFound => { _errorMessage = "Search not found."; }, + validation => { _errorMessage = validation.Message; }, + unauthorized => { _errorMessage = "Session expired."; }, + forbidden => { _errorMessage = "Access denied."; }, + error => { _errorMessage = error.Message; } + ); + } + else + { + _search = new ClientSearchViewModel + { + Status = "New", + UserName = await AuthStateProvider.GetUsernameAsync() ?? "", + Criteria = new SearchCriteriaViewModel() + }; + } + + DetectSearchType(); + } + finally + { + _isLoading = false; + } +} +``` + +**Step 4: Add error message field** + +```csharp +private string? _errorMessage; +``` + +**Step 5: Verify build** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 6: Commit** + +```bash +git add src/JdeScoping.Client/Pages/SearchEdit.razor +git commit -m "feat(client): migrate SearchEdit.razor load methods to API clients" +``` + +--- + +### Task 7: Migrate SearchEdit.razor - Part 2 (Save and Download Methods) + +**Files:** +- Modify: `src/JdeScoping.Client/Pages/SearchEdit.razor` + +**Step 1: Update SubmitSearchInternalAsync method** + +Replace the save call: + +```csharp +private async Task SubmitSearchInternalAsync() +{ + var confirmed = await DialogService.Confirm("Are you sure you want to submit the search?", "Confirm Submit", new ConfirmOptions + { + OkButtonText = "Submit", + CancelButtonText = "Cancel" + }); + + if (confirmed != true) + { + return; + } + + _isSubmitting = true; + try + { + var result = await SearchApi.CreateSearchAsync(_search.ToCore()); + result.Switch( + id => { NavigationManager.NavigateTo($"/search/{id}"); }, + notFound => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Search not found."); }, + validation => { NotificationService.Notify(NotificationSeverity.Error, "Validation Error", validation.Message); }, + unauthorized => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Session expired."); }, + forbidden => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Access denied."); }, + error => { NotificationService.Notify(NotificationSeverity.Error, "Error", error.Message); } + ); + } + finally + { + _isSubmitting = false; + } +} +``` + +**Step 2: Update DownloadResultsAsync method** + +Replace with: + +```csharp +private async Task DownloadResultsAsync() +{ + var result = await SearchApi.GetResultsAsync(_search.Id); + result.Switch( + bytes => + { + if (bytes.Length > 0) + { + _ = JSRuntime.InvokeVoidAsync("downloadFile", $"search_results_{_search.Id}.xlsx", bytes); + NotificationService.Notify(NotificationSeverity.Success, "Download", "Results downloaded successfully."); + } + else + { + NotificationService.Notify(NotificationSeverity.Warning, "Download", "No results available to download."); + } + }, + notFound => { NotificationService.Notify(NotificationSeverity.Warning, "Download", "Results not found."); }, + validation => { NotificationService.Notify(NotificationSeverity.Error, "Error", validation.Message); }, + unauthorized => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Session expired."); }, + forbidden => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Access denied."); }, + error => { NotificationService.Notify(NotificationSeverity.Error, "Error", error.Message); } + ); +} +``` + +**Step 3: Verify build and commit** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` + +```bash +git add src/JdeScoping.Client/Pages/SearchEdit.razor +git commit -m "feat(client): migrate SearchEdit.razor save and download methods" +``` + +--- + +### Task 8: Migrate Login.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Pages/Login.razor` + +**Step 1: Update inject statement** + +Change: +```razor +@inject IAuthService AuthService +``` +To: +```razor +@inject IAuthApiClient AuthApi +@inject ICryptoService CryptoService +``` + +**Step 2: Update HandleLoginAsync method** + +Replace with: + +```csharp +private async Task HandleLoginAsync() +{ + _isLoading = true; + _errorMessage = null; + + try + { + // Encrypt credentials + var encryptedData = await CryptoService.EncryptLoginAsync(_loginModel); + var request = new EncryptedLoginRequest(encryptedData); + + var result = await AuthApi.LoginAsync(request); + result.Switch( + loginResult => + { + if (loginResult.Success && loginResult.User is not null) + { + var returnUrl = string.IsNullOrEmpty(ReturnUrl) ? "/" : ReturnUrl; + NavigationManager.NavigateTo(returnUrl); + } + else + { + _errorMessage = loginResult.ErrorMessage ?? "Login failed. Please check your credentials."; + } + }, + notFound => { _errorMessage = "Authentication service not found."; }, + validation => { _errorMessage = validation.Message; }, + unauthorized => { _errorMessage = "Invalid credentials."; }, + forbidden => { _errorMessage = "Access denied."; }, + error => { _errorMessage = error.Message; } + ); + } + catch (Exception ex) + { + _errorMessage = $"An error occurred: {ex.Message}"; + } + finally + { + _isLoading = false; + } +} +``` + +**Step 3: Add required using** + +```razor +@using JdeScoping.Core.Models +``` + +**Step 4: Verify build and commit** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` + +```bash +git add src/JdeScoping.Client/Pages/Login.razor +git commit -m "feat(client): migrate Login.razor to IAuthApiClient" +``` + +--- + +### Task 9: Migrate MainLayout.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Layout/MainLayout.razor` + +**Step 1: Update inject statement** + +Change: +```razor +@inject IAuthService AuthService +``` +To: +```razor +@inject IAuthApiClient AuthApi +``` + +**Step 2: Update LogoutAsync method** + +Replace with: + +```csharp +private async Task LogoutAsync() +{ + await AuthApi.LogoutAsync(); + NavigationManager.NavigateTo("/login"); +} +``` + +**Step 3: Verify build and commit** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` + +```bash +git add src/JdeScoping.Client/Layout/MainLayout.razor +git commit -m "feat(client): migrate MainLayout.razor to IAuthApiClient" +``` + +--- + +### Task 10: Migrate ItemNumberFilterPanel.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Components/FilterPanels/ItemNumberFilterPanel.razor` + +**Step 1: Update inject statements** + +Change: +```razor +@inject ILookupService LookupService +@inject IFileService FileService +``` +To: +```razor +@inject ILookupApiClient LookupApi +@inject IFileApiClient FileApi +``` + +**Step 2: Update OnSearchAsync method** + +Replace with: + +```csharp +private async Task OnSearchAsync(LoadDataArgs args) +{ + if (!string.IsNullOrEmpty(args.Filter) && args.Filter.Length >= 3) + { + var result = await LookupApi.FindItemsAsync(args.Filter); + result.Switch( + items => { _searchResults = items.ToList(); }, + _ => { _searchResults = []; }, + _ => { _searchResults = []; }, + _ => { _searchResults = []; }, + _ => { _searchResults = []; }, + _ => { _searchResults = []; } + ); + } + else + { + _searchResults = []; + } +} +``` + +**Step 3: Update DownloadTemplateAsync method** + +Replace with: + +```csharp +private async Task DownloadTemplateAsync() +{ + var result = await FileApi.DownloadItemsTemplateAsync(Items.AsReadOnly()); + result.Switch( + bytes => { _ = JSRuntime.InvokeVoidAsync("downloadFile", "items_template.xlsx", bytes); }, + _ => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Template not found."); }, + validation => { NotificationService.Notify(NotificationSeverity.Error, "Error", validation.Message); }, + _ => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Session expired."); }, + _ => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Access denied."); }, + error => { NotificationService.Notify(NotificationSeverity.Error, "Error", error.Message); } + ); +} +``` + +**Step 4: Update OnFileSelected method** + +Replace with: + +```csharp +private async Task OnFileSelected(InputFileChangeEventArgs e) +{ + if (e.File == null) return; + + _isUploading = true; + try + { + using var stream = e.File.OpenReadStream(maxAllowedSize: 10 * 1024 * 1024); + var result = await FileApi.UploadItemsAsync(stream, e.File.Name); + + result.Switch( + items => + { + Items.Clear(); + Items.AddRange(items); + _ = ItemsChanged.InvokeAsync(Items); + NotificationService.Notify(NotificationSeverity.Success, "Upload Complete", $"Loaded {items.Count} items."); + }, + _ => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Upload endpoint not found."); }, + validation => { NotificationService.Notify(NotificationSeverity.Error, "Upload Failed", validation.Message); }, + _ => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Session expired."); }, + _ => { NotificationService.Notify(NotificationSeverity.Error, "Error", "Access denied."); }, + error => { NotificationService.Notify(NotificationSeverity.Error, "Upload Failed", error.Message); } + ); + } + catch (Exception ex) + { + NotificationService.Notify(NotificationSeverity.Error, "Upload Failed", ex.Message); + } + finally + { + _isUploading = false; + } +} +``` + +**Step 5: Verify build and commit** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` + +```bash +git add src/JdeScoping.Client/Components/FilterPanels/ItemNumberFilterPanel.razor +git commit -m "feat(client): migrate ItemNumberFilterPanel to API clients" +``` + +--- + +### Task 11: Migrate WorkCenterFilterPanel.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Components/FilterPanels/WorkCenterFilterPanel.razor` + +**Step 1: Update inject statement** + +Change `@inject ILookupService LookupService` to `@inject ILookupApiClient LookupApi` + +**Step 2: Update search method to use ApiResult pattern** + +Similar to ItemNumberFilterPanel, update the autocomplete search to use `result.Switch()`. + +**Step 3: Verify build and commit** + +```bash +git add src/JdeScoping.Client/Components/FilterPanels/WorkCenterFilterPanel.razor +git commit -m "feat(client): migrate WorkCenterFilterPanel to ILookupApiClient" +``` + +--- + +### Task 12: Migrate ProfitCenterFilterPanel.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Components/FilterPanels/ProfitCenterFilterPanel.razor` + +**Step 1: Update inject statement** + +Change `@inject ILookupService LookupService` to `@inject ILookupApiClient LookupApi` + +**Step 2: Update search method to use ApiResult pattern** + +**Step 3: Verify build and commit** + +```bash +git add src/JdeScoping.Client/Components/FilterPanels/ProfitCenterFilterPanel.razor +git commit -m "feat(client): migrate ProfitCenterFilterPanel to ILookupApiClient" +``` + +--- + +### Task 13: Migrate OperatorFilterPanel.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Components/FilterPanels/OperatorFilterPanel.razor` + +**Step 1: Update inject statement and add using** + +Change `@inject ILookupService LookupService` to `@inject ILookupApiClient LookupApi` + +Add: `@using JdeScoping.Client.Extensions` + +**Step 2: Update search method** + +Use `result.Switch()` and map JdeUserViewModel to OperatorViewModel using `.ToClientOperatorList()`. + +**Step 3: Verify build and commit** + +```bash +git add src/JdeScoping.Client/Components/FilterPanels/OperatorFilterPanel.razor +git commit -m "feat(client): migrate OperatorFilterPanel to ILookupApiClient" +``` + +--- + +### Task 14: Migrate WorkOrderFilterPanel.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Components/FilterPanels/WorkOrderFilterPanel.razor` + +**Step 1: Update inject statement** + +Change `@inject IFileService FileService` to `@inject IFileApiClient FileApi` + +**Step 2: Update download and upload methods** + +Use `result.Switch()` pattern for `DownloadWorkOrdersTemplateAsync` and `UploadWorkOrdersAsync`. + +**Step 3: Verify build and commit** + +```bash +git add src/JdeScoping.Client/Components/FilterPanels/WorkOrderFilterPanel.razor +git commit -m "feat(client): migrate WorkOrderFilterPanel to IFileApiClient" +``` + +--- + +### Task 15: Migrate ComponentLotFilterPanel.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Components/FilterPanels/ComponentLotFilterPanel.razor` + +**Step 1: Update inject statement** + +Change `@inject IFileService FileService` to `@inject IFileApiClient FileApi` + +**Step 2: Update download and upload methods** + +Use `result.Switch()` pattern for `DownloadComponentLotsTemplateAsync` and `UploadComponentLotsAsync`. + +**Step 3: Verify build and commit** + +```bash +git add src/JdeScoping.Client/Components/FilterPanels/ComponentLotFilterPanel.razor +git commit -m "feat(client): migrate ComponentLotFilterPanel to IFileApiClient" +``` + +--- + +### Task 16: Migrate PartOperationFilterPanel.razor + +**Files:** +- Modify: `src/JdeScoping.Client/Components/FilterPanels/PartOperationFilterPanel.razor` + +**Step 1: Update inject statement** + +Change `@inject IFileService FileService` to `@inject IFileApiClient FileApi` + +**Step 2: Update download and upload methods** + +Use `result.Switch()` pattern for `DownloadPartOperationsTemplateAsync` and `UploadPartOperationsAsync`. + +**Step 3: Verify build and commit** + +```bash +git add src/JdeScoping.Client/Components/FilterPanels/PartOperationFilterPanel.razor +git commit -m "feat(client): migrate PartOperationFilterPanel to IFileApiClient" +``` + +--- + +### Task 17: Update Program.cs - Remove Old Service Registrations + +**Files:** +- Modify: `src/JdeScoping.Client/Program.cs` + +**Step 1: Remove old service registrations** + +Remove these lines: +```csharp +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +``` + +Keep: +```csharp +builder.Services.AddScoped(); // Keep temporarily - AuthService has crypto logic +``` + +**Step 2: Verify build** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded with no references to old services + +**Step 3: Commit** + +```bash +git add src/JdeScoping.Client/Program.cs +git commit -m "chore(client): remove old service registrations" +``` + +--- + +### Task 18: Delete Old Service Files + +**Files:** +- Delete: `src/JdeScoping.Client/Services/ISearchService.cs` +- Delete: `src/JdeScoping.Client/Services/SearchService.cs` +- Delete: `src/JdeScoping.Client/Services/ILookupService.cs` +- Delete: `src/JdeScoping.Client/Services/LookupService.cs` +- Delete: `src/JdeScoping.Client/Services/IFileService.cs` +- Delete: `src/JdeScoping.Client/Services/FileService.cs` + +**Step 1: Delete files** + +```bash +rm src/JdeScoping.Client/Services/ISearchService.cs +rm src/JdeScoping.Client/Services/SearchService.cs +rm src/JdeScoping.Client/Services/ILookupService.cs +rm src/JdeScoping.Client/Services/LookupService.cs +rm src/JdeScoping.Client/Services/IFileService.cs +rm src/JdeScoping.Client/Services/FileService.cs +``` + +**Step 2: Verify build** + +Run: `dotnet build src/JdeScoping.Client/JdeScoping.Client.csproj` +Expected: Build succeeded + +**Step 3: Commit** + +```bash +git add -A +git commit -m "chore(client): delete old service files replaced by API clients" +``` + +--- + +### Task 19: Final Build Verification + +**Files:** +- All modified files + +**Step 1: Clean and rebuild entire solution** + +Run: +```bash +dotnet clean +dotnet build +``` +Expected: Build succeeded with 0 errors + +**Step 2: Run any available tests** + +Run: +```bash +dotnet test +``` +Expected: All tests pass + +**Step 3: Commit if any fixes needed** + +```bash +git add -A +git commit -m "fix(client): address any build issues from migration" +``` + +--- + +### Task 20: Update _Imports.razor + +**Files:** +- Modify: `src/JdeScoping.Client/_Imports.razor` + +**Step 1: Add using for extensions namespace** + +Add: +```razor +@using JdeScoping.Client.Extensions +``` + +**Step 2: Verify build and commit** + +```bash +git add src/JdeScoping.Client/_Imports.razor +git commit -m "chore(client): add Extensions namespace to global imports" +``` + +--- + +## Summary + +This plan migrates 12 Blazor components from old `I*Service` interfaces to new `I*ApiClient` interfaces: +- 5 pages (Searches, SearchQueue, SearchEdit, Login, MainLayout) +- 7 filter panels (ItemNumber, WorkCenter, ProfitCenter, Operator, WorkOrder, ComponentLot, PartOperation) + +Key changes: +1. Global 401 handling via AuthRedirectHandler +2. View model mapping via extension methods +3. ApiResult pattern with Switch() for error handling +4. Delete 6 old service files after migration diff --git a/PLANS/2026-01-06-old-etl-removal-design.md b/PLANS/2026-01-06-old-etl-removal-design.md new file mode 100644 index 0000000..b865aa1 --- /dev/null +++ b/PLANS/2026-01-06-old-etl-removal-design.md @@ -0,0 +1,721 @@ +# Old ETL Removal Design + +## Goal + +Remove the legacy ETL implementation (Fetchers, MergeConfigurations, BulkMerge services, SourceGenerator) and wire the existing orchestration layer to use the new `EtlPipeline` system. + +## Background + +The codebase has two parallel ETL implementations: +- **OLD:** `IDataFetcher` → `BulkMergeHelper` → `IMergeConfiguration` with source-generated `IDataReader` implementations +- **NEW:** `EtlPipeline` with `IImportSource` → `IDataTransformer` → `IImportDestination` + +The new pipeline is working well. The old implementation can be removed. + +## Design Decisions + +1. **Keep orchestration layer** - `DataSyncService`, `SyncOrchestrator`, `ScheduleChecker` remain; only internals change +2. **Keep tracking** - `DataUpdateRepository` preserved; new pipeline writes sync timestamps +3. **JSON config-driven pipelines** - Pipeline definitions loaded from JSON files at runtime (not compiled code) +4. **Builder pattern for factory** - `IEtlPipelineFactory` uses fluent builder: `.ForTable().WithMode().Build()` +5. **Generic DbQuerySource** - Single source class with connection type specified in config (not separate Oracle/Sybase classes) +6. **Conditional merge support** - Extend `DbBulkMergeDestination` with `UpdateWhen` condition +7. **Relative time offsets** - MinDt parameter uses TimeSpan format (e.g., `"-7.00:00:00"`) computed at runtime +8. **Config table names** - Factory uses table names exactly as defined in config (e.g., `WorkOrder_Curr`) +9. **MisData post-processing** - Convert `MisDataPostProcessor` to SQL post-script in pipeline +10. **Sync mode mapping** - Daily and Hourly both map to `incremental` mode; ScheduleChecker can override offset at runtime +11. **Parameter mapping** - Config defines parameter mappings for provider-specific syntax (`:dateUpdated` vs `@MinDt`) and format conversions (JDE Julian) +12. **Destination override** - Base destination config, sync modes can override destination type/settings +13. **Exclude list for updates** - Default: update all non-match columns; config can specify `excludeFromUpdate` for exceptions +14. **Implement new first** - Build new factory/sources/config before deleting old code to keep build working +15. **Config as content file** - `pipelines.json` copied to output directory, loaded from disk at runtime +16. **PrePurge/ReIndex as scripts** - PrePurge becomes pre-script (TRUNCATE), ReIndex becomes post-script +17. **Partial merge for overrides** - Mode-specific destination config merges with base (only specified fields override) +18. **Generic parameters** - Support arbitrary parameters with source types: `offset`, `static`, `runtime` +19. **Configurable timezone** - JDE Julian conversion uses configurable timezone (UTC or local) +20. **Fail fast on missing config** - Factory throws if requested sync mode not defined in config +21. **Runtime parameters deferred** - Only `offset` and `static` parameter sources supported; throw if `runtime` used +22. **Pipeline config owns PrePurge/ReIndex** - Remove schedule flags; pipeline config is single source of truth +23. **JSON camelCase** - Use `JsonSerializerOptions` with `PropertyNameCaseInsensitive = true` + +## Files to Delete + +### Source Generator Project (entire project) +``` +src/JdeScoping.DataSync.SourceGenerators/ +├── DataReaderGenerator.cs +├── IsExternalInit.cs +└── JdeScoping.DataSync.SourceGenerators.csproj +``` + +### DataSync Source Files (~32 files) +``` +src/JdeScoping.DataSync/ +├── BulkCopyTypeRegistry.cs +├── Contracts/ +│ ├── IBulkMergeHelper.cs +│ ├── IDataFetcher.cs +│ ├── IDataReaderFactory.cs +│ ├── IMergeConfiguration.cs +│ ├── IMergeConfigurationRegistry.cs +│ ├── IPostProcessor.cs +│ └── ISchemaValidator.cs +├── Configuration/MergeConfigurations/ +│ ├── BranchMergeConfiguration.cs +│ ├── ItemMergeConfiguration.cs +│ ├── JdeUserMergeConfiguration.cs +│ ├── LotMergeConfiguration.cs +│ ├── LotUsageMergeConfiguration.cs +│ ├── MisDataMergeConfiguration.cs +│ ├── ProfitCenterMergeConfiguration.cs +│ ├── WorkCenterMergeConfiguration.cs +│ └── WorkOrderMergeConfiguration.cs +├── Exceptions/BulkMergeException.cs +├── Fetchers/ +│ ├── Cms/CmsMisDataFetcher.cs +│ └── Jde/ +│ ├── JdeBranchFetcher.cs +│ ├── JdeItemFetcher.cs +│ ├── JdeLotFetcher.cs +│ ├── JdeLotUsageFetcher.cs +│ ├── JdeProfitCenterFetcher.cs +│ ├── JdeUserFetcher.cs +│ ├── JdeWorkCenterFetcher.cs +│ └── JdeWorkOrderFetcher.cs +├── Models/ +│ ├── ColumnSchema.cs +│ └── MergeResult.cs +└── Services/ + ├── BulkMergeHelper.cs + ├── ExpressionParser.cs + ├── MergeConfigurationRegistry.cs + ├── MergeSqlBuilder.cs + ├── MisDataPostProcessor.cs + └── SchemaValidator.cs +``` + +### Test Files (~8 files) +``` +tests/JdeScoping.DataSync.Tests/ +├── Services/ +│ ├── BulkMergeHelperTests.cs +│ ├── ExpressionParserTests.cs +│ ├── MergeConfigurationRegistryTests.cs +│ ├── MergeSqlBuilderTests.cs +│ └── SchemaValidatorTests.cs +└── TableSyncOperationTests.cs + +tests/JdeScoping.DataSync.IntegrationTests/ +├── BulkMergeHelperTests.cs +└── TableSyncOperationTests.cs +``` + +### Integration Test Infrastructure (entire folder) +``` +tests/JdeScoping.DataSync.IntegrationTests/Infrastructure/ +├── TestDataReaderFactory.cs +├── BulkMergeTestEntityDataReader.cs +├── BulkMergeTestEntity.cs +├── TestDatabaseInitializer.cs +├── TestDbConnectionFactory.cs +├── TestDataGenerator.cs +└── SqlServerFixture.cs +``` + +Consider removing entire `tests/JdeScoping.DataSync.IntegrationTests/` project if all tests are obsolete. + +## Files to Modify + +### 1. TableSyncOperation.cs - Major Rewrite +**Current:** Uses `IDataFetcher`, `IBulkMergeHelper`, `IMergeConfiguration` +**New:** Uses `IEtlPipelineFactory` to get and execute pipelines + +### 2. DependencyInjection.cs - Remove Old Registrations +**Remove:** +- `using JdeScoping.DataSync.Generated;` statement +- All `IDataFetcher` registrations +- All `IMergeConfiguration` registrations +- `IBulkMergeHelper`, `IDataReaderFactory`, `ISchemaValidator` +- `IMergeConfigurationRegistry` +- `IPostProcessor`, `MisDataPostProcessor` +- Named fetcher registrations +- `DataReaderFactory` registration + +**Keep:** +- `DataSyncService`, `ISyncOrchestrator`, `IScheduleChecker` +- `IDataUpdateRepository` +- Health check and metrics + +**Add:** +- `IEtlPipelineFactory` registration +- `PipelineOptions` configuration binding + +### 3. DataSourceConfig.cs - Remove Unused Properties +Remove these properties: +- `FetcherTypeName` +- `PostProcessorTypeName` +- `PrepurgeData` (now in pipeline config) +- `ReIndexData` (now in pipeline config) + +### 4. appsettings.json / appsettings.Development.json +Remove `FetcherTypeName` and `PostProcessorTypeName` from data source configurations + +### 5. JdeScoping.slnx +Remove `JdeScoping.DataSync.SourceGenerators` project reference + +### 6. JdeScoping.DataSync.csproj +- Remove reference to SourceGenerators project +- Remove `InternalsVisibleTo` for integration tests if project removed + +### 7. Tests to Update (not delete) +- `ScheduleCheckerTests.cs` - Update test fixtures to remove FetcherTypeName +- `SyncOrchestratorTests.cs` - Update test fixtures to remove FetcherTypeName + +## Files to Create + +### 1. Pipeline Configuration Files + +Pipeline definitions stored in JSON, copied to output directory at build time. + +**Location:** `src/JdeScoping.DataSync/Pipelines/pipelines.json` + +**Project file entry:** +```xml + + + PreserveNewest + + +``` + +**Complete Schema Example:** +```json +{ + "settings": { + "timezone": "UTC" // or "Local" - used for JDE Julian conversion + }, + "pipelines": { + "WorkOrder_Curr": { + "source": { + "connection": "jde", + "query": "SELECT WADOCO, WADC0J, ... FROM PRODDTA.F4801 WHERE UPMJ >= :dateUpdated", + "parameters": { + "minDt": { + "name": ":dateUpdated", + "format": "jdeJulian", + "source": "offset" + } + } + }, + "syncModes": { + "mass": { + "minDtOffset": "-365.00:00:00", + "prePurge": true, + "reIndex": true + }, + "incremental": { + "minDtOffset": "-1.00:00:00" + } + }, + "transformers": [ + { "type": "jdeDate", "columns": ["OrderDate", "CompletionDate", "StartDate"] }, + { "type": "columnRename", "mappings": { "WADOCO": "OrderNumber", "WADC0J": "Branch" } } + ], + "destination": { + "table": "WorkOrder_Curr", + "matchColumns": ["OrderNumber"], + "excludeFromUpdate": ["CreatedDate"] + } + }, + "MisData": { + "source": { + "connection": "cms", + "query": "SELECT ... FROM MIS_DATA WHERE LastUpdate >= @MinDt", + "parameters": { + "minDt": { + "name": "@MinDt", + "source": "offset" + } + } + }, + "syncModes": { + "mass": { + "minDtOffset": "-365.00:00:00", + "prePurge": true, + "destination": { "type": "bulkImport" } + }, + "incremental": { + "minDtOffset": "-7.00:00:00", + "destination": { "type": "bulkMerge" } + } + }, + "destination": { + "table": "MisData", + "matchColumns": ["MisDataId"] + }, + "postScripts": [ + "UPDATE MisData SET ProcessedFlag = 1 WHERE ProcessedFlag IS NULL" + ] + } + } +} +``` + +**PrePurge/ReIndex behavior:** +- `prePurge: true` → Factory adds pre-script: `TRUNCATE TABLE [TableName]` +- `reIndex: true` → Factory adds post-script: `ALTER INDEX ALL ON [TableName] REBUILD` + +### 2. Pipeline Configuration Models + +**PipelinesRoot.cs** - Root config structure +```csharp +namespace JdeScoping.DataSync.Configuration; + +public record PipelinesRoot( + PipelineSettings Settings, + Dictionary Pipelines); + +public record PipelineSettings( + string Timezone = "UTC"); // "UTC" or "Local" +``` + +**PipelineConfig.cs** +```csharp +namespace JdeScoping.DataSync.Configuration; + +public record PipelineConfig( + SourceConfig Source, + Dictionary SyncModes, + List? Transformers, + DestinationConfig Destination, + List? PreScripts, + List? PostScripts); + +public record SourceConfig( + string Connection, // "jde", "cms", "lotfinder" + string Query, + Dictionary? Parameters); + +public record ParameterConfig( + string Name, // Provider-specific: ":dateUpdated" or "@MinDt" + string? Format, // "jdeJulian", "jdeTime", null for DateTime + string Source = "offset", // "offset", "static", "runtime" + string? Value); // For static source + +public record SyncModeConfig( + string? MinDtOffset, // TimeSpan format: "-7.00:00:00" + bool PrePurge = false, + bool ReIndex = false, + string? UpdateWhen = null, // Conditional update expression + DestinationOverride? Destination = null); // Override base destination (partial merge) + +public record DestinationOverride( + string? Type, // "bulkImport" or "bulkMerge" + List? MatchColumns, // Override match columns + List? ExcludeFromUpdate); // Override exclude list + +public record TransformerConfig( + string Type, + List? Columns, + Dictionary? Mappings); + +public record DestinationConfig( + string Table, + List? MatchColumns, // For merge operations + List? ExcludeFromUpdate); // Columns to skip on update +``` + +**PipelineOptions.cs** +```csharp +namespace JdeScoping.DataSync.Options; + +public class PipelineOptions +{ + public const string SectionName = "Pipelines"; + public string ConfigPath { get; set; } = "Pipelines/pipelines.json"; +} +``` + +**Config loading with validation:** +```csharp +private static readonly JsonSerializerOptions JsonOptions = new() +{ + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true +}; + +private PipelinesRoot LoadPipelineConfigs(string configPath) +{ + // Resolve path relative to assembly location (handles both debug and publish) + var assemblyDir = Path.GetDirectoryName(typeof(EtlPipelineFactory).Assembly.Location)!; + var fullPath = Path.Combine(assemblyDir, configPath); + + if (!File.Exists(fullPath)) + throw new FileNotFoundException($"Pipeline config not found: {fullPath}"); + + var json = File.ReadAllText(fullPath); + var root = JsonSerializer.Deserialize(json, JsonOptions) + ?? throw new InvalidOperationException("Failed to deserialize pipeline config"); + + // Validate all pipelines have required sync modes + foreach (var (name, config) in root.Pipelines) + { + if (!config.SyncModes.ContainsKey("mass")) + throw new InvalidOperationException($"Pipeline '{name}' missing 'mass' sync mode"); + if (!config.SyncModes.ContainsKey("incremental")) + throw new InvalidOperationException($"Pipeline '{name}' missing 'incremental' sync mode"); + + // Validate no runtime parameters (not yet supported) + if (config.Source.Parameters != null) + { + foreach (var (paramName, paramConfig) in config.Source.Parameters) + { + if (paramConfig.Source == "runtime") + throw new NotSupportedException( + $"Pipeline '{name}' parameter '{paramName}': runtime source not yet supported"); + } + } + } + + return root; +} +``` + +### 3. IEtlPipelineFactory.cs (Builder Pattern) + +```csharp +namespace JdeScoping.DataSync.Contracts; + +public interface IEtlPipelineFactory +{ + IEtlPipelineBuilder ForTable(string tableName); +} + +public interface IEtlPipelineBuilder +{ + IEtlPipelineBuilder WithMode(SyncMode mode); + IEtlPipelineBuilder WithMinimumDate(DateTime? minDt); // Override config offset + EtlPipeline Build(); +} + +// Note: No WithPrePurge/WithReIndex - pipeline config is source of truth + +public enum SyncMode +{ + Mass, // Full refresh - uses bulkImport by default + Incremental // Delta sync - uses bulkMerge by default +} +``` + +### 4. EtlPipelineFactory.cs + +```csharp +namespace JdeScoping.DataSync.Services; + +public class EtlPipelineFactory : IEtlPipelineFactory +{ + private readonly IDbConnectionFactory _connectionFactory; + private readonly Dictionary _configs; + + public EtlPipelineFactory( + IDbConnectionFactory connectionFactory, + IOptions options) + { + _connectionFactory = connectionFactory; + _configs = LoadPipelineConfigs(options.Value.ConfigPath); + } + + public IEtlPipelineBuilder ForTable(string tableName) + { + if (!_configs.TryGetValue(tableName, out var config)) + throw new ArgumentException($"No pipeline configured for table: {tableName}"); + + return new PipelineBuilder(_connectionFactory, tableName, config); + } + + private class PipelineBuilder : IEtlPipelineBuilder + { + private readonly IDbConnectionFactory _connectionFactory; + private readonly string _tableName; + private readonly PipelineConfig _config; + private readonly PipelineSettings _settings; + private SyncMode _mode = SyncMode.Incremental; + private DateTime? _minDtOverride; + + public EtlPipeline Build() + { + var modeKey = _mode == SyncMode.Mass ? "mass" : "incremental"; + var modeConfig = _config.SyncModes[modeKey]; // Already validated at load + + // Compute MinDt from offset or override + var minDt = _minDtOverride ?? ComputeMinDt(modeConfig.MinDtOffset); + + // Create source with parameter substitution + var source = CreateSource(_config.Source, minDt); + + // Determine destination type (mode override > default by mode) + var destType = modeConfig.Destination?.Type + ?? (_mode == SyncMode.Mass ? "bulkImport" : "bulkMerge"); + var destination = CreateDestination(destType, _config.Destination, modeConfig); + + // Build pipeline with scripts + var builder = new EtlPipelineBuilder() + .WithName(_tableName) + .WithSource(source) + .WithDestination(destination); + + // Add pre-scripts: config scripts first, then prePurge + foreach (var script in _config.PreScripts ?? []) + builder.WithPreScript(new SqlScriptRunner(_connectionFactory, script)); + if (modeConfig.PrePurge) + builder.WithPreScript(new SqlScriptRunner(_connectionFactory, + $"TRUNCATE TABLE [{_config.Destination.Table}]")); + + // Add post-scripts: reIndex first, then config scripts + if (modeConfig.ReIndex) + builder.WithPostScript(new SqlScriptRunner(_connectionFactory, + $"ALTER INDEX ALL ON [{_config.Destination.Table}] REBUILD")); + foreach (var script in _config.PostScripts ?? []) + builder.WithPostScript(new SqlScriptRunner(_connectionFactory, script)); + + // Add transformers + foreach (var t in _config.Transformers ?? []) + builder.WithTransformer(CreateTransformer(t)); + + return builder.Build(); + } + } +} +``` + +### 5. DbQuerySource.cs (Generic) + +```csharp +namespace JdeScoping.DataSync.Etl.Sources; + +/// +/// Generic database query source that works with any connection type. +/// +public class DbQuerySource : IImportSource +{ + private readonly IDbConnectionFactory _connectionFactory; + private readonly string _connectionType; // "jde", "cms", "lotfinder" + private readonly string _query; + private readonly Dictionary _parameters; + + public string SourceName => $"DbQuery:{_connectionType}"; + + public DbQuerySource( + IDbConnectionFactory connectionFactory, + string connectionType, + string query, + Dictionary? parameters = null) + { + _connectionFactory = connectionFactory; + _connectionType = connectionType; + _query = query; + _parameters = parameters ?? new(); + } + + public async Task ReadDataAsync(CancellationToken ct) + { + var connection = _connectionType switch + { + "jde" => await _connectionFactory.CreateJdeConnectionAsync(), + "cms" => await _connectionFactory.CreateCmsConnectionAsync(), + "lotfinder" => await _connectionFactory.CreateLotFinderConnectionAsync(), + _ => throw new ArgumentException($"Unknown connection type: {_connectionType}") + }; + + var command = connection.CreateCommand(); + command.CommandText = _query; + + foreach (var (name, value) in _parameters) + { + var param = command.CreateParameter(); + param.ParameterName = name; + param.Value = value; + command.Parameters.Add(param); + } + + return await command.ExecuteReaderAsync(CommandBehavior.CloseConnection, ct); + } +} +``` + +### 6. Parameter Format Converters + +```csharp +namespace JdeScoping.DataSync.Services; + +public class ParameterFormatConverter +{ + private readonly TimeZoneInfo _timezone; + + public ParameterFormatConverter(string timezone) + { + _timezone = timezone.ToUpperInvariant() switch + { + "UTC" => TimeZoneInfo.Utc, + "LOCAL" => TimeZoneInfo.Local, + _ => TimeZoneInfo.FindSystemTimeZoneById(timezone) + }; + } + + public object Convert(DateTime minDt, string? format) + { + // Convert to configured timezone + var adjusted = TimeZoneInfo.ConvertTime(minDt, _timezone); + + return format switch + { + "jdeJulian" => ToJdeJulianDate(adjusted), + "jdeTime" => ToJdeTime(adjusted), + null => adjusted, + _ => throw new ArgumentException($"Unknown format: {format}") + }; + } + + private static int ToJdeJulianDate(DateTime date) + { + // JDE Julian: CYYDDD where C=century (0=19xx, 1=20xx), YY=year, DDD=day of year + int century = date.Year >= 2000 ? 1 : 0; + int year = date.Year % 100; + int dayOfYear = date.DayOfYear; + return century * 100000 + year * 1000 + dayOfYear; + } + + private static int ToJdeTime(DateTime time) + { + // JDE Time: HHMMSS + return time.Hour * 10000 + time.Minute * 100 + time.Second; + } +} +``` + +### 7. DbBulkMergeDestination Extension + +Extend to support conditional updates and exclude columns: + +```csharp +// Updated constructor +public DbBulkMergeDestination( + IDbConnectionFactory connectionFactory, + string tableName, + string[] matchColumns, + string[]? excludeFromUpdate = null, + string? updateCondition = null) // e.g., "source.LastUpdateDt > target.LastUpdateDt" +``` + +Modify MERGE SQL generation: +```sql +MERGE INTO [Target] AS target +USING #TempTable AS source +ON target.OrderNumber = source.OrderNumber +WHEN MATCHED AND source.LastUpdateDt > target.LastUpdateDt THEN -- updateCondition + UPDATE SET + target.Col1 = source.Col1, + -- excludeFromUpdate columns omitted +WHEN NOT MATCHED THEN + INSERT (...) + VALUES (...); +``` + +## TableSyncOperation.cs - Error Handling + +`EtlPipeline.ExecuteAsync` returns `PipelineResult` with `Success=false` on failure (doesn't throw). +`TableSyncOperation` must check `Success` and throw to keep `DataUpdate` records correct: + +```csharp +var pipeline = _pipelineFactory + .ForTable(config.TableName) + .WithMode(updateTask.IsMassUpdate ? SyncMode.Mass : SyncMode.Incremental) + .WithMinimumDate(updateTask.MinimumDt) // ScheduleChecker can override + .Build(); + +var result = await pipeline.ExecuteAsync(ct); + +if (!result.Success) + throw new InvalidOperationException($"Pipeline failed for {config.TableName}: {result.ErrorMessage}"); +``` + +## Files to Keep (No Changes) + +- `DataSyncService.cs` - Background service host +- `SyncOrchestrator.cs` - Orchestration logic +- `ScheduleChecker.cs` - Schedule checking logic (provides mode and can override offset) +- `DataUpdateRepository.cs` - Sync timestamp tracking +- `DataSyncHealthCheck.cs` - Health monitoring +- `DataSyncMetrics.cs` - Telemetry +- `DataSyncActivitySource.cs` - Tracing + +## Test Files to Keep (with updates) + +- `SyncOrchestratorTests.cs` - Update fixtures to remove FetcherTypeName +- `ScheduleCheckerTests.cs` - Update fixtures to remove FetcherTypeName +- `DataSyncServiceTests.cs` +- `DataSyncHealthCheckTests.cs` + +## New Tests to Create + +- `EtlPipelineFactoryTests.cs` - Test config loading and pipeline building +- `DbQuerySourceTests.cs` - Test connection type switching and parameter handling +- `ParameterFormatConverterTests.cs` - Test JDE Julian/time conversions +- `DbBulkMergeDestinationTests.cs` - Test UpdateWhen and excludeFromUpdate + +## Validation & Precedence Rules + +### Required Fields (fail at config load) +- `source.connection` - must be "jde", "cms", or "lotfinder" +- `source.query` - must be non-empty +- `destination.table` - must be non-empty +- `syncModes.mass` and `syncModes.incremental` - both required + +### Precedence Rules +1. **MinDt**: `WithMinimumDate()` override > config `minDtOffset` computation +2. **PrePurge/ReIndex**: Removed from builder; pipeline config is only source +3. **Scripts order**: Config `preScripts` run first, then generated prePurge script; generated reIndex script runs first, then config `postScripts` +4. **Destination merge**: Mode-specific fields override base; missing fields inherit from base + +### Parameter Resolution +- `offset`: Computed from `minDtOffset` + current time; format conversion applied +- `static`: Value taken from config `value` field; must be present; no format conversion +- `runtime`: Throws `NotSupportedException` (deferred) + +## Risk Assessment + +**Low risk:** +- New ETL pipeline already working and tested +- Orchestration layer unchanged (just different internals) +- Clear separation between old and new code + +**Medium risk:** +- Generic DbQuerySource with multiple connection types - need testing +- Conditional merge (UpdateWhen) is new - need testing +- JSON config loading is new - need validation +- Parameter format conversion (JDE Julian) - need testing + +## Migration Path + +**Phase 1: Build New (keep build working)** +1. Create pipeline config models and options +2. Create `DbQuerySource` (generic) +3. Extend `DbBulkMergeDestination` with UpdateWhen and excludeFromUpdate +4. Create `ParameterFormatConverter` +5. Create `IEtlPipelineFactory` and `EtlPipelineFactory` +6. Create `pipelines.json` config file +7. Register new services in DI (alongside old) + +**Phase 2: Wire Up** +8. Update `TableSyncOperation` to use `IEtlPipelineFactory` +9. Update `DependencyInjection.cs` to wire new factory +10. Test end-to-end with new pipeline + +**Phase 3: Clean Up** +11. Delete old source files (Fetchers, MergeConfigurations, BulkMerge services) +12. Delete old contracts +13. Delete SourceGenerator project +14. Update solution file +15. Update tests (remove FetcherTypeName references) +16. Delete obsolete test files and infrastructure diff --git a/PLANS/2026-01-06-old-etl-removal-implementation.md b/PLANS/2026-01-06-old-etl-removal-implementation.md new file mode 100644 index 0000000..cebd613 --- /dev/null +++ b/PLANS/2026-01-06-old-etl-removal-implementation.md @@ -0,0 +1,824 @@ +# Old ETL Removal Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Remove legacy ETL implementation and wire orchestration to use new EtlPipeline with JSON config. + +**Architecture:** Three-phase migration - build new infrastructure first, wire up, then clean up old code. + +**Tech Stack:** .NET 10, System.Text.Json, EtlPipeline + +**Working Directory:** All paths are relative to `NEW/` folder. Run `cd /Users/dohertj2/Desktop/JdeScopingTool/NEW` before starting. + +--- + +## Phase 1: Build New Infrastructure + +### Task 1: Create Pipeline Configuration Models + +**Files:** +- Create: `src/JdeScoping.DataSync/Configuration/PipelinesRoot.cs` +- Create: `src/JdeScoping.DataSync/Configuration/PipelineConfig.cs` +- Create: `src/JdeScoping.DataSync/Options/PipelineOptions.cs` + +**Step 1: Create PipelinesRoot.cs** +```csharp +namespace JdeScoping.DataSync.Configuration; + +public record PipelinesRoot( + PipelineSettings? Settings, // Optional - defaults applied if missing + Dictionary Pipelines) +{ + public PipelineSettings EffectiveSettings => Settings ?? new PipelineSettings(); +} + +public record PipelineSettings( + string Timezone = "UTC"); +``` + +**Step 2: Create PipelineConfig.cs** +```csharp +namespace JdeScoping.DataSync.Configuration; + +public record PipelineConfig( + SourceConfig Source, + Dictionary SyncModes, + List? Transformers, + DestinationConfig Destination, + List? PreScripts, + List? PostScripts); + +public record SourceConfig( + string Connection, + string Query, + Dictionary? Parameters); + +public record ParameterConfig( + string Name, + string? Format, + string Source = "offset", + string? Value); + +public record SyncModeConfig( + string? MinDtOffset, + bool PrePurge = false, + bool ReIndex = false, + string? UpdateWhen = null, + DestinationOverride? Destination = null); + +public record DestinationOverride( + string? Type, + List? MatchColumns, + List? ExcludeFromUpdate); + +public record TransformerConfig( + string Type, + List? Columns, + Dictionary? Mappings); + +public record DestinationConfig( + string Table, + List? MatchColumns, + List? ExcludeFromUpdate); +``` + +**Step 3: Create PipelineOptions.cs** +```csharp +namespace JdeScoping.DataSync.Options; + +public class PipelineOptions +{ + public const string SectionName = "Pipelines"; + public string ConfigPath { get; set; } = "Pipelines/pipelines.json"; +} +``` + +**Step 4: Build to verify** +```bash +dotnet build src/JdeScoping.DataSync/JdeScoping.DataSync.csproj +``` + +**Step 5: Commit** +```bash +git add -A && git commit -m "feat(datasync): add pipeline configuration models" +``` + +--- + +### Task 2: Create ParameterFormatConverter + +**Files:** +- Create: `src/JdeScoping.DataSync/Services/ParameterFormatConverter.cs` +- Create: `tests/JdeScoping.DataSync.Tests/Services/ParameterFormatConverterTests.cs` + +**Step 1: Create ParameterFormatConverter.cs** +```csharp +namespace JdeScoping.DataSync.Services; + +public class ParameterFormatConverter +{ + private readonly TimeZoneInfo _timezone; + + public ParameterFormatConverter(string timezone) + { + _timezone = timezone.ToUpperInvariant() switch + { + "UTC" => TimeZoneInfo.Utc, + "LOCAL" => TimeZoneInfo.Local, + _ => TimeZoneInfo.FindSystemTimeZoneById(timezone) + }; + } + + public object Convert(DateTime value, string? format) + { + var adjusted = TimeZoneInfo.ConvertTime(value, _timezone); + + return format?.ToLowerInvariant() switch + { + "jdejulian" => ToJdeJulianDate(adjusted), + "jdetime" => ToJdeTime(adjusted), + null => adjusted, + _ => throw new ArgumentException($"Unknown format: {format}") + }; + } + + public static int ToJdeJulianDate(DateTime date) + { + int century = date.Year >= 2000 ? 1 : 0; + int year = date.Year % 100; + int dayOfYear = date.DayOfYear; + return century * 100000 + year * 1000 + dayOfYear; + } + + public static int ToJdeTime(DateTime time) + { + return time.Hour * 10000 + time.Minute * 100 + time.Second; + } +} +``` + +**Step 2: Create tests** +```csharp +namespace JdeScoping.DataSync.Tests.Services; + +public class ParameterFormatConverterTests +{ + [Fact] + public void ToJdeJulianDate_Year2024Day100_Returns124100() + { + var date = new DateTime(2024, 4, 9); // Day 100 + var result = ParameterFormatConverter.ToJdeJulianDate(date); + result.ShouldBe(124100); + } + + [Fact] + public void ToJdeJulianDate_Year1999Day365_Returns99365() + { + var date = new DateTime(1999, 12, 31); + var result = ParameterFormatConverter.ToJdeJulianDate(date); + result.ShouldBe(99365); + } + + [Fact] + public void ToJdeTime_143025_Returns143025() + { + var time = new DateTime(2024, 1, 1, 14, 30, 25); + var result = ParameterFormatConverter.ToJdeTime(time); + result.ShouldBe(143025); + } + + [Fact] + public void Convert_WithUtcTimezone_UsesUtc() + { + var converter = new ParameterFormatConverter("UTC"); + var utcTime = DateTime.SpecifyKind(new DateTime(2024, 4, 9, 12, 0, 0), DateTimeKind.Utc); + var result = converter.Convert(utcTime, "jdeJulian"); + result.ShouldBe(124100); + } +} +``` + +**Step 3: Run tests** +```bash +dotnet test tests/JdeScoping.DataSync.Tests --filter "ParameterFormatConverterTests" +``` + +**Step 4: Commit** +```bash +git add -A && git commit -m "feat(datasync): add ParameterFormatConverter with JDE date/time support" +``` + +--- + +### Task 3: Extend DbQuerySource for Multiple Connections + +**Files:** +- Modify: `src/JdeScoping.DataSync/Etl/Sources/DbQuerySource.cs` +- Modify: `tests/JdeScoping.DataSync.Tests/Etl/Sources/DbQuerySourceTests.cs` + +**Note:** DbQuerySource already exists but only supports LotFinder. Extend it to support JDE and CMS connections. + +**Step 1: Update DbQuerySource.cs** +```csharp +using System.Data; +using System.Data.Common; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Contracts; + +namespace JdeScoping.DataSync.Etl.Sources; + +public class DbQuerySource : IImportSource +{ + private readonly IDbConnectionFactory _connectionFactory; + private readonly string _connectionType; + private readonly string _query; + private readonly Dictionary _parameters; + private DbConnection? _connection; + + public string SourceName => $"DbQuery:{_connectionType}"; + + public DbQuerySource( + IDbConnectionFactory connectionFactory, + string connectionType, + string query, + Dictionary? parameters = null) + { + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + _connectionType = connectionType?.ToLowerInvariant() + ?? throw new ArgumentNullException(nameof(connectionType)); + _query = query ?? throw new ArgumentNullException(nameof(query)); + _parameters = parameters ?? new Dictionary(); + + if (_connectionType is not ("jde" or "cms" or "lotfinder")) + throw new ArgumentException($"Unknown connection type: {connectionType}"); + } + + public async Task ReadDataAsync(CancellationToken cancellationToken = default) + { + _connection = _connectionType switch + { + "jde" => await _connectionFactory.CreateJdeConnectionAsync(), + "cms" => await _connectionFactory.CreateCmsConnectionAsync(), + "lotfinder" => await _connectionFactory.CreateLotFinderConnectionAsync(), + _ => throw new InvalidOperationException($"Unknown connection type: {_connectionType}") + }; + + var command = _connection.CreateCommand(); + command.CommandText = _query; + + foreach (var (name, value) in _parameters) + { + var param = command.CreateParameter(); + param.ParameterName = name; + param.Value = value ?? DBNull.Value; + command.Parameters.Add(param); + } + + return await command.ExecuteReaderAsync(CommandBehavior.CloseConnection, cancellationToken); + } + + public async ValueTask DisposeAsync() + { + if (_connection != null) + { + await _connection.DisposeAsync(); + _connection = null; + } + } +} +``` + +**Step 2: Create basic tests** +```csharp +namespace JdeScoping.DataSync.Tests.Etl.Sources; + +public class DbQuerySourceTests +{ + [Theory] + [InlineData("jde")] + [InlineData("cms")] + [InlineData("lotfinder")] + public void Constructor_ValidConnectionType_Succeeds(string connectionType) + { + var factory = Substitute.For(); + var source = new DbQuerySource(factory, connectionType, "SELECT 1"); + source.SourceName.ShouldBe($"DbQuery:{connectionType}"); + } + + [Fact] + public void Constructor_InvalidConnectionType_Throws() + { + var factory = Substitute.For(); + Should.Throw(() => + new DbQuerySource(factory, "invalid", "SELECT 1")); + } + + [Fact] + public void Constructor_NullQuery_Throws() + { + var factory = Substitute.For(); + Should.Throw(() => + new DbQuerySource(factory, "jde", null!)); + } +} +``` + +**Step 3: Run tests** +```bash +dotnet test tests/JdeScoping.DataSync.Tests --filter "DbQuerySourceTests" +``` + +**Step 4: Commit** +```bash +git add -A && git commit -m "feat(datasync): add generic DbQuerySource for JDE/CMS/LotFinder" +``` + +--- + +### Task 4: Extend DbBulkMergeDestination + +**Files:** +- Modify: `src/JdeScoping.DataSync/Etl/Destinations/DbBulkMergeDestination.cs` +- Create/Modify: `tests/JdeScoping.DataSync.Tests/Etl/Destinations/DbBulkMergeDestinationTests.cs` + +**Step 1: Add excludeFromUpdate and updateCondition parameters** + +Add to constructor: +```csharp +public DbBulkMergeDestination( + IDbConnectionFactory connectionFactory, + string tableName, + string[] matchColumns, + string[]? excludeFromUpdate = null, + string? updateCondition = null) +``` + +**Step 2: Modify MERGE SQL generation to use new parameters** + +Update the WHEN MATCHED clause to include condition and exclude columns. + +**Step 3: Add tests for new functionality** + +**Step 4: Run tests** +```bash +dotnet test tests/JdeScoping.DataSync.Tests --filter "DbBulkMergeDestinationTests" +``` + +**Step 5: Commit** +```bash +git add -A && git commit -m "feat(datasync): extend DbBulkMergeDestination with excludeFromUpdate and updateCondition" +``` + +--- + +### Task 5: Create IEtlPipelineFactory and Contracts + +**Files:** +- Create: `src/JdeScoping.DataSync/Contracts/IEtlPipelineFactory.cs` +- Create: `src/JdeScoping.DataSync/Contracts/SyncMode.cs` + +**Step 1: Create IEtlPipelineFactory.cs** +```csharp +namespace JdeScoping.DataSync.Contracts; + +public interface IEtlPipelineFactory +{ + IEtlPipelineBuilder ForTable(string tableName); +} + +public interface IEtlPipelineBuilder +{ + IEtlPipelineBuilder WithMode(SyncMode mode); + IEtlPipelineBuilder WithMinimumDate(DateTime? minDt); + EtlPipeline Build(); +} +``` + +**Step 2: Create SyncMode.cs** +```csharp +namespace JdeScoping.DataSync.Contracts; + +public enum SyncMode +{ + Mass, + Incremental +} +``` + +**Step 3: Build to verify** +```bash +dotnet build src/JdeScoping.DataSync/JdeScoping.DataSync.csproj +``` + +**Step 4: Commit** +```bash +git add -A && git commit -m "feat(datasync): add IEtlPipelineFactory and SyncMode contracts" +``` + +--- + +### Task 6: Create EtlPipelineFactory + +**Files:** +- Create: `src/JdeScoping.DataSync/Services/EtlPipelineFactory.cs` +- Create: `tests/JdeScoping.DataSync.Tests/Services/EtlPipelineFactoryTests.cs` + +**Step 1: Create EtlPipelineFactory.cs** + +Implement the factory with: +- Config loading with validation +- PipelineBuilder inner class +- Source/destination/transformer creation methods + +**Step 2: Add tests for config loading and validation** + +**Step 3: Run tests** +```bash +dotnet test tests/JdeScoping.DataSync.Tests --filter "EtlPipelineFactoryTests" +``` + +**Step 4: Commit** +```bash +git add -A && git commit -m "feat(datasync): add EtlPipelineFactory with JSON config support" +``` + +--- + +### Task 7: Create pipelines.json Config File + +**Files:** +- Create: `src/JdeScoping.DataSync/Pipelines/pipelines.json` +- Modify: `src/JdeScoping.DataSync/JdeScoping.DataSync.csproj` + +**Step 1: Extract data from existing merge configurations** + +Read existing merge configs to extract for each table: +- `MatchOn` → `matchColumns` +- `UpdateColumns` / `InsertColumns` → derive `excludeFromUpdate` +- `UpdateWhen` → `updateCondition` + +Files to reference: +- `src/JdeScoping.DataSync/Configuration/MergeConfigurations/*.cs` +- `src/JdeScoping.DataSync/Fetchers/Jde/*.cs` (for queries) +- `src/JdeScoping.DataSync/Fetchers/Cms/*.cs` (for CMS query) + +**Step 2: Create Pipelines directory and pipelines.json** + +Create config for all 9 tables: +- WorkOrder_Curr +- Lot +- LotUsage +- Item +- WorkCenter +- ProfitCenter +- JdeUser +- Branch +- MisData + +**Important:** For MisData, add the post-processing SQL as a postScript: +```json +"postScripts": [ + "UPDATE MisData SET ProcessedFlag = 1 WHERE ProcessedFlag IS NULL" +] +``` +This replaces the MisDataPostProcessor class. + +**Step 3: Add Content item to csproj** +```xml + + + PreserveNewest + + +``` + +**Step 4: Build to verify config copies** +```bash +dotnet build src/JdeScoping.DataSync/JdeScoping.DataSync.csproj +ls src/JdeScoping.DataSync/bin/Debug/net10.0/Pipelines/ +``` + +**Step 5: Commit** +```bash +git add -A && git commit -m "feat(datasync): add pipelines.json config for all sync tables" +``` + +--- + +## Phase 2: Wire Up + +### Task 8: Update DependencyInjection.cs + +**Files:** +- Modify: `src/JdeScoping.DataSync/DependencyInjection.cs` + +**Step 1: Add new registrations (alongside old for now)** +```csharp +// Add pipeline factory +services.AddOptions() + .Bind(configuration.GetSection(PipelineOptions.SectionName)); +services.AddSingleton(); +``` + +**Step 2: Build to verify** +```bash +dotnet build src/JdeScoping.DataSync/JdeScoping.DataSync.csproj +``` + +**Step 3: Commit** +```bash +git add -A && git commit -m "feat(datasync): register EtlPipelineFactory in DI" +``` + +--- + +### Task 9: Update TableSyncOperation + +**Files:** +- Modify: `src/JdeScoping.DataSync/Services/TableSyncOperation.cs` + +**Step 1: Inject IEtlPipelineFactory** + +**Step 2: Replace old sync logic with pipeline execution** +```csharp +var pipeline = _pipelineFactory + .ForTable(config.TableName) + .WithMode(updateTask.IsMassUpdate ? SyncMode.Mass : SyncMode.Incremental) + .WithMinimumDate(updateTask.MinimumDt) + .Build(); + +var result = await pipeline.ExecuteAsync(cancellationToken); + +if (!result.Success) + throw new InvalidOperationException($"Pipeline failed for {config.TableName}: {result.ErrorMessage}"); + +// Important: Pass row count to DataUpdateRepository for metrics +var recordCount = result.TotalRows; // Use this for DataUpdate record +``` + +**Step 3: Build to verify** +```bash +dotnet build src/JdeScoping.DataSync/JdeScoping.DataSync.csproj +``` + +**Step 4: Commit** +```bash +git add -A && git commit -m "feat(datasync): wire TableSyncOperation to use EtlPipelineFactory" +``` + +--- + +--- + +## Phase 3: Clean Up + +**Important:** Tasks in Phase 3 must be executed in order. DataSourceConfig changes come AFTER test and appsettings updates to avoid broken builds. + +### Task 10: Remove Old DI Registrations + +**Files:** +- Modify: `src/JdeScoping.DataSync/DependencyInjection.cs` + +**Step 1: Remove old registrations** +- Remove `using JdeScoping.DataSync.Generated;` +- Remove all `IDataFetcher` registrations +- Remove all `IMergeConfiguration` registrations +- Remove `IBulkMergeHelper`, `IDataReaderFactory`, `ISchemaValidator` +- Remove `IMergeConfigurationRegistry` +- Remove `IPostProcessor`, `MisDataPostProcessor` +- Remove named fetcher registrations + +**Step 2: Build to verify** +```bash +dotnet build src/JdeScoping.DataSync/JdeScoping.DataSync.csproj +``` + +**Step 3: Commit** +```bash +git add -A && git commit -m "refactor(datasync): remove old ETL DI registrations" +``` + +--- + +### Task 11: Delete Old Source Files + +**Files to delete:** +- `BulkCopyTypeRegistry.cs` +- `Contracts/IBulkMergeHelper.cs` +- `Contracts/IDataFetcher.cs` +- `Contracts/IDataReaderFactory.cs` +- `Contracts/IMergeConfiguration.cs` +- `Contracts/IMergeConfigurationRegistry.cs` +- `Contracts/IPostProcessor.cs` +- `Contracts/ISchemaValidator.cs` +- `Configuration/MergeConfigurations/` (all 9 files) +- `Exceptions/BulkMergeException.cs` +- `Fetchers/` (all files) +- `Models/ColumnSchema.cs` +- `Models/MergeResult.cs` +- `Services/BulkMergeHelper.cs` +- `Services/ExpressionParser.cs` +- `Services/MergeConfigurationRegistry.cs` +- `Services/MergeSqlBuilder.cs` +- `Services/MisDataPostProcessor.cs` +- `Services/SchemaValidator.cs` + +**Step 1: Delete files** +```bash +rm src/JdeScoping.DataSync/BulkCopyTypeRegistry.cs +rm -rf src/JdeScoping.DataSync/Contracts/IBulkMergeHelper.cs +# ... etc +``` + +**Step 2: Build to verify** +```bash +dotnet build src/JdeScoping.DataSync/JdeScoping.DataSync.csproj +``` + +**Step 3: Commit** +```bash +git add -A && git commit -m "refactor(datasync): delete old ETL source files" +``` + +--- + +### Task 12: Delete Integration Tests Project + +**Files:** +- Delete: `tests/JdeScoping.DataSync.IntegrationTests/` (entire project) +- Modify: `JdeScoping.slnx` + +**Note:** Must delete integration tests BEFORE removing SourceGenerator, as integration tests reference generated code. + +**Step 1: Remove project from solution** +```bash +dotnet sln JdeScoping.slnx remove tests/JdeScoping.DataSync.IntegrationTests/JdeScoping.DataSync.IntegrationTests.csproj +``` + +**Step 2: Delete project folder** +```bash +rm -rf tests/JdeScoping.DataSync.IntegrationTests +``` + +**Step 3: Build to verify** +```bash +dotnet build JdeScoping.slnx +``` + +**Step 4: Commit** +```bash +git add -A && git commit -m "refactor(datasync): remove obsolete integration tests project" +``` + +--- + +### Task 13: Delete SourceGenerator Project + +**Files:** +- Delete: `src/JdeScoping.DataSync.SourceGenerators/` (entire project) +- Modify: `JdeScoping.slnx` +- Modify: `src/JdeScoping.DataSync/JdeScoping.DataSync.csproj` + +**Step 1: Remove project reference from DataSync.csproj** + +**Step 2: Remove project from solution** +```bash +dotnet sln JdeScoping.slnx remove src/JdeScoping.DataSync.SourceGenerators/JdeScoping.DataSync.SourceGenerators.csproj +``` + +**Step 3: Delete project folder** +```bash +rm -rf src/JdeScoping.DataSync.SourceGenerators +``` + +**Step 4: Build to verify** +```bash +dotnet build JdeScoping.slnx +``` + +**Step 5: Commit** +```bash +git add -A && git commit -m "refactor(datasync): remove SourceGenerator project" +``` + +--- + +### Task 14: Delete Old Unit Test Files + +**Files to delete:** +- `tests/JdeScoping.DataSync.Tests/Services/BulkMergeHelperTests.cs` +- `tests/JdeScoping.DataSync.Tests/Services/ExpressionParserTests.cs` +- `tests/JdeScoping.DataSync.Tests/Services/MergeConfigurationRegistryTests.cs` +- `tests/JdeScoping.DataSync.Tests/Services/MergeSqlBuilderTests.cs` +- `tests/JdeScoping.DataSync.Tests/Services/SchemaValidatorTests.cs` +- `tests/JdeScoping.DataSync.Tests/TableSyncOperationTests.cs` + +**Step 1: Delete test files** + +**Step 2: Run remaining tests** +```bash +dotnet test tests/JdeScoping.DataSync.Tests +``` + +**Step 3: Commit** +```bash +git add -A && git commit -m "refactor(datasync): delete obsolete test files" +``` + +--- + +### Task 15: Update Remaining Tests + +**Files:** +- Modify: `tests/JdeScoping.DataSync.Tests/ScheduleCheckerTests.cs` +- Modify: `tests/JdeScoping.DataSync.Tests/SyncOrchestratorTests.cs` + +**Step 1: Remove FetcherTypeName references from test fixtures** + +**Step 2: Run tests** +```bash +dotnet test tests/JdeScoping.DataSync.Tests +``` + +**Step 3: Commit** +```bash +git add -A && git commit -m "refactor(datasync): update tests to remove FetcherTypeName" +``` + +--- + +### Task 16: Update appsettings Files + +**Files:** +- Modify: `src/JdeScoping.Host/appsettings.json` +- Modify: `src/JdeScoping.Host/appsettings.Development.json` + +**Step 1: Remove obsolete properties from DataSources config** +- FetcherTypeName +- PostProcessorTypeName +- PrepurgeData +- ReIndexData + +**Step 2: Build and run to verify** +```bash +dotnet build src/JdeScoping.Host/JdeScoping.Host.csproj +``` + +**Step 3: Commit** +```bash +git add -A && git commit -m "refactor(datasync): remove obsolete appsettings properties" +``` + +--- + +### Task 17: Update DataSourceConfig + +**Files:** +- Modify: `src/JdeScoping.DataSync/Options/DataSourceConfig.cs` + +**Note:** This task comes AFTER test and appsettings updates to avoid broken builds. + +**Step 1: Remove obsolete properties** +- FetcherTypeName +- PostProcessorTypeName +- PrepurgeData +- ReIndexData + +**Step 2: Build to verify** +```bash +dotnet build JdeScoping.slnx +``` + +**Step 3: Commit** +```bash +git add -A && git commit -m "refactor(datasync): remove obsolete DataSourceConfig properties" +``` + +--- + +### Task 18: Final Verification + +**Step 1: Full build** +```bash +dotnet build JdeScoping.slnx +``` + +**Step 2: Run all tests** +```bash +dotnet test JdeScoping.slnx +``` + +**Step 3: Commit any fixes** + +**Step 4: Create summary commit** +```bash +git add -A && git commit -m "feat(datasync): complete migration to JSON-configured ETL pipelines + +- Remove legacy Fetchers, MergeConfigurations, BulkMerge services +- Remove SourceGenerator project +- Add EtlPipelineFactory with JSON config +- Add DbQuerySource for JDE/CMS/LotFinder connections +- Extend DbBulkMergeDestination with excludeFromUpdate and updateCondition +- Wire TableSyncOperation to use new pipeline factory +- Update all tests and configuration" +```