feat: add startup config validation and document ConfigManager pipeline editor

Add ConfigurationValidationRunner with IConfigurationValidator interface for
validating required settings at startup. Includes SecureStore and LDAP validators.
Expand ConfigManager with pipeline editing UI, dialogs, and step editors.
Update documentation with config validation guidance.
This commit is contained in:
Joseph Doherty
2026-01-21 17:47:15 -05:00
parent ceb63bfefb
commit e5fe2f06e9
88 changed files with 4995 additions and 201 deletions
@@ -23,6 +23,7 @@ public interface IDataUpdateRepository
/// <param name="sourceData">Source data identifier.</param>
/// <param name="tableName">Target table name.</param>
/// <param name="updateType">Type of update.</param>
/// <param name="parameters">Optional JSON string of parameters used during the sync operation.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The ID of the created record.</returns>
Task<int> StartUpdateAsync(
@@ -30,6 +31,7 @@ public interface IDataUpdateRepository
string sourceData,
string tableName,
UpdateTypes updateType,
string? parameters = null,
CancellationToken cancellationToken = default);
/// <summary>
@@ -50,7 +50,8 @@ SELECT cte.ID,
cte.EndDT,
cte.UpdateType,
cte.WasSuccessful,
cte.NumberRecords
cte.NumberRecords,
cte.Parameters
FROM DU_CTE cte
WHERE cte.RN = 1";
@@ -68,12 +69,13 @@ WHERE cte.RN = 1";
string sourceData,
string tableName,
UpdateTypes updateType,
string? parameters = null,
CancellationToken cancellationToken = default)
{
const string sql = @"
INSERT INTO dbo.DataUpdate (SourceSystem, SourceData, TableName, UpdateType, StartDT, NumberRecords, WasSuccessful)
INSERT INTO dbo.DataUpdate (SourceSystem, SourceData, TableName, UpdateType, StartDT, NumberRecords, WasSuccessful, Parameters)
OUTPUT INSERTED.ID
VALUES (@sourceSystem, @sourceData, @tableName, @updateType, GETUTCDATE(), @inProgressMarker, 0)";
VALUES (@sourceSystem, @sourceData, @tableName, @updateType, GETUTCDATE(), @inProgressMarker, 0, @parameters)";
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(cancellationToken);
var id = await connection.ExecuteScalarAsync<int>(
@@ -84,7 +86,8 @@ VALUES (@sourceSystem, @sourceData, @tableName, @updateType, GETUTCDATE(), @inPr
sourceData,
tableName,
updateType = (int)updateType,
inProgressMarker = InProgressMarker
inProgressMarker = InProgressMarker,
parameters
},
commandTimeout: 30);
@@ -268,12 +271,12 @@ FROM LastSuccessful";
{
var sql = updateType.HasValue
? @"SELECT TOP (@count) du.Id, du.SourceSystem, du.SourceData, du.TableName,
du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords
du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords, du.Parameters
FROM dbo.DataUpdate du
WHERE du.TableName = @tableName AND du.UpdateType = @updateType
ORDER BY du.StartDt DESC"
: @"SELECT TOP (@count) du.Id, du.SourceSystem, du.SourceData, du.TableName,
du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords
du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords, du.Parameters
FROM dbo.DataUpdate du
WHERE du.TableName = @tableName
ORDER BY du.StartDt DESC";
@@ -299,7 +302,7 @@ WITH LastRuns AS (
FROM dbo.DataUpdate du
WHERE du.TableName = @tableName
)
SELECT Id, SourceSystem, SourceData, TableName, StartDt, EndDt, UpdateType, WasSuccessful, NumberRecords
SELECT Id, SourceSystem, SourceData, TableName, StartDt, EndDt, UpdateType, WasSuccessful, NumberRecords, Parameters
FROM LastRuns
WHERE RN = 1";
@@ -64,12 +64,16 @@ public class TableSyncOperation : ITableSyncOperation
_metrics.RecordOperationStarted(task.TableName, task.UpdateType.ToString());
// Build parameters JSON for tracking
var parametersJson = BuildParametersJson(task);
// Log start of data update
var updateId = await _updateRepository.StartUpdateAsync(
task.SourceSystem,
task.SourceData,
task.TableName,
task.UpdateType,
parametersJson,
cancellationToken);
long recordCount = 0;
@@ -133,6 +137,24 @@ public class TableSyncOperation : ITableSyncOperation
}
}
/// <summary>
/// Builds a JSON string of parameters for the DataUpdate record.
/// </summary>
private static string? BuildParametersJson(DataUpdateTask task)
{
var parametersDict = new Dictionary<string, object?>
{
["OperationId"] = task.OperationId.ToString()
};
if (task.MinimumDt.HasValue)
{
parametersDict["MinimumDt"] = task.MinimumDt.Value.ToString("O");
}
return System.Text.Json.JsonSerializer.Serialize(parametersDict);
}
/// <summary>
/// Core sync logic that uses the ETL pipeline.
/// </summary>