feat: implement ETL pipeline redesign and ConfigManager improvements

- Add pipeline registry with JSON-based configuration and hot-reload support
- Implement manual sync request feature with API, client UI, and database
- Improve ConfigManager: connection string dropdown in pipeline editor,
  step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
This commit is contained in:
Joseph Doherty
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
@@ -0,0 +1,24 @@
{
"name": "WorkOrderTime_Curr",
"isEnabled": true,
"isManualOnly": false,
"massSyncIntervalMinutes": 10080,
"dailySyncIntervalMinutes": 1440,
"hourlySyncIntervalMinutes": 60,
"source": {
"connection": "jde",
"query": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot WHERE (wot.DATEUPDATED_WTUPMJ > :dateUpdated OR (wot.DATEUPDATED_WTUPMJ = :dateUpdated AND wot.TIMEOFDAY_WTTDAY >= :timeUpdated))",
"massQuery": "SELECT wot.UNIQUEKEYIDINTERNAL_WTUKID AS UniqueID, TRIM(wot.COSTCENTERALT_WTMMCU) AS BranchCode, wot.DOCUMENTORDERINVOICEE_WTDOCO AS WorkOrderNumber, wot.SEQUENCENOOPERATIONS_WTOPSQ AS StepNumber, wot.ADDRESSNUMBER_WTAN8 AS AddressNumber, wot.DTFORGLANDVOUCH1_WTDGL AS GlDate, wot.DATEUPDATED_WTUPMJ AS DateUpdated, wot.TIMEOFDAY_WTTDAY AS TimeUpdated FROM JDESTAGE.F31122_VIEW wot",
"parameters": {
"dateUpdated": { "name": ":dateUpdated", "format": "jdeJulian", "source": "offset" },
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"destination": {
"table": "WorkOrderTime_Curr",
"matchColumns": ["UniqueID"],
"excludeFromUpdate": ["UniqueID", "LastUpdateDt"]
},
"preScripts": [],
"postScripts": []
}