feat: add startup config validation and document ConfigManager pipeline editor
Add ConfigurationValidationRunner with IConfigurationValidator interface for validating required settings at startup. Includes SecureStore and LDAP validators. Expand ConfigManager with pipeline editing UI, dialogs, and step editors. Update documentation with config validation guidance.
This commit is contained in:
@@ -23,6 +23,7 @@ public interface IDataUpdateRepository
|
||||
/// <param name="sourceData">Source data identifier.</param>
|
||||
/// <param name="tableName">Target table name.</param>
|
||||
/// <param name="updateType">Type of update.</param>
|
||||
/// <param name="parameters">Optional JSON string of parameters used during the sync operation.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The ID of the created record.</returns>
|
||||
Task<int> StartUpdateAsync(
|
||||
@@ -30,6 +31,7 @@ public interface IDataUpdateRepository
|
||||
string sourceData,
|
||||
string tableName,
|
||||
UpdateTypes updateType,
|
||||
string? parameters = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -50,7 +50,8 @@ SELECT cte.ID,
|
||||
cte.EndDT,
|
||||
cte.UpdateType,
|
||||
cte.WasSuccessful,
|
||||
cte.NumberRecords
|
||||
cte.NumberRecords,
|
||||
cte.Parameters
|
||||
FROM DU_CTE cte
|
||||
WHERE cte.RN = 1";
|
||||
|
||||
@@ -68,12 +69,13 @@ WHERE cte.RN = 1";
|
||||
string sourceData,
|
||||
string tableName,
|
||||
UpdateTypes updateType,
|
||||
string? parameters = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
const string sql = @"
|
||||
INSERT INTO dbo.DataUpdate (SourceSystem, SourceData, TableName, UpdateType, StartDT, NumberRecords, WasSuccessful)
|
||||
INSERT INTO dbo.DataUpdate (SourceSystem, SourceData, TableName, UpdateType, StartDT, NumberRecords, WasSuccessful, Parameters)
|
||||
OUTPUT INSERTED.ID
|
||||
VALUES (@sourceSystem, @sourceData, @tableName, @updateType, GETUTCDATE(), @inProgressMarker, 0)";
|
||||
VALUES (@sourceSystem, @sourceData, @tableName, @updateType, GETUTCDATE(), @inProgressMarker, 0, @parameters)";
|
||||
|
||||
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(cancellationToken);
|
||||
var id = await connection.ExecuteScalarAsync<int>(
|
||||
@@ -84,7 +86,8 @@ VALUES (@sourceSystem, @sourceData, @tableName, @updateType, GETUTCDATE(), @inPr
|
||||
sourceData,
|
||||
tableName,
|
||||
updateType = (int)updateType,
|
||||
inProgressMarker = InProgressMarker
|
||||
inProgressMarker = InProgressMarker,
|
||||
parameters
|
||||
},
|
||||
commandTimeout: 30);
|
||||
|
||||
@@ -268,12 +271,12 @@ FROM LastSuccessful";
|
||||
{
|
||||
var sql = updateType.HasValue
|
||||
? @"SELECT TOP (@count) du.Id, du.SourceSystem, du.SourceData, du.TableName,
|
||||
du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords
|
||||
du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords, du.Parameters
|
||||
FROM dbo.DataUpdate du
|
||||
WHERE du.TableName = @tableName AND du.UpdateType = @updateType
|
||||
ORDER BY du.StartDt DESC"
|
||||
: @"SELECT TOP (@count) du.Id, du.SourceSystem, du.SourceData, du.TableName,
|
||||
du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords
|
||||
du.StartDt, du.EndDt, du.UpdateType, du.WasSuccessful, du.NumberRecords, du.Parameters
|
||||
FROM dbo.DataUpdate du
|
||||
WHERE du.TableName = @tableName
|
||||
ORDER BY du.StartDt DESC";
|
||||
@@ -299,7 +302,7 @@ WITH LastRuns AS (
|
||||
FROM dbo.DataUpdate du
|
||||
WHERE du.TableName = @tableName
|
||||
)
|
||||
SELECT Id, SourceSystem, SourceData, TableName, StartDt, EndDt, UpdateType, WasSuccessful, NumberRecords
|
||||
SELECT Id, SourceSystem, SourceData, TableName, StartDt, EndDt, UpdateType, WasSuccessful, NumberRecords, Parameters
|
||||
FROM LastRuns
|
||||
WHERE RN = 1";
|
||||
|
||||
|
||||
@@ -64,12 +64,16 @@ public class TableSyncOperation : ITableSyncOperation
|
||||
|
||||
_metrics.RecordOperationStarted(task.TableName, task.UpdateType.ToString());
|
||||
|
||||
// Build parameters JSON for tracking
|
||||
var parametersJson = BuildParametersJson(task);
|
||||
|
||||
// Log start of data update
|
||||
var updateId = await _updateRepository.StartUpdateAsync(
|
||||
task.SourceSystem,
|
||||
task.SourceData,
|
||||
task.TableName,
|
||||
task.UpdateType,
|
||||
parametersJson,
|
||||
cancellationToken);
|
||||
|
||||
long recordCount = 0;
|
||||
@@ -133,6 +137,24 @@ public class TableSyncOperation : ITableSyncOperation
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds a JSON string of parameters for the DataUpdate record.
|
||||
/// </summary>
|
||||
private static string? BuildParametersJson(DataUpdateTask task)
|
||||
{
|
||||
var parametersDict = new Dictionary<string, object?>
|
||||
{
|
||||
["OperationId"] = task.OperationId.ToString()
|
||||
};
|
||||
|
||||
if (task.MinimumDt.HasValue)
|
||||
{
|
||||
parametersDict["MinimumDt"] = task.MinimumDt.Value.ToString("O");
|
||||
}
|
||||
|
||||
return System.Text.Json.JsonSerializer.Serialize(parametersDict);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Core sync logic that uses the ETL pipeline.
|
||||
/// </summary>
|
||||
|
||||
Reference in New Issue
Block a user