refactor: relocate options classes to dedicated Options folders

Move configuration options from Core/DataAccess/DataSync/ExcelIO to
dedicated Options folders within each project for better organization.
Update all references and tests accordingly.
This commit is contained in:
Joseph Doherty
2026-01-03 08:55:08 -05:00
parent 3cb73eb09f
commit ec4c8fab87
52 changed files with 4628 additions and 202 deletions
@@ -1,6 +1,6 @@
using System.Text.Json.Serialization;
using JdeScoping.Api.Hubs;
using JdeScoping.Core.Options;
using JdeScoping.Api.Options;
using Microsoft.AspNetCore.Authentication.Cookies;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
@@ -1,4 +1,4 @@
namespace JdeScoping.Core.Options;
namespace JdeScoping.Api.Options;
/// <summary>
/// Authentication configuration options
@@ -10,12 +10,6 @@ public class AuthOptions
/// </summary>
public const string SectionName = "Auth";
/// <summary>
/// Enable fake authentication for development.
/// When true, any credentials are accepted.
/// </summary>
public bool UseFakeAuth { get; set; } = false;
/// <summary>
/// Name of the authentication cookie.
/// </summary>
@@ -25,10 +19,4 @@ public class AuthOptions
/// Cookie expiration in minutes (default: 8 hours).
/// </summary>
public int CookieExpirationMinutes { get; set; } = 480;
/// <summary>
/// Optional list of usernames that bypass group check.
/// Use sparingly for admin/testing purposes.
/// </summary>
public string[] AdminBypassUsers { get; set; } = [];
}
@@ -1,32 +0,0 @@
namespace JdeScoping.Core.Options;
/// <summary>
/// Configuration options for data synchronization background jobs.
/// </summary>
public class DataSyncOptions
{
/// <summary>
/// Configuration section name in appsettings.json.
/// </summary>
public const string SectionName = "DataSync";
/// <summary>
/// Cron schedule for mass (full) data refresh. Empty string disables the schedule.
/// </summary>
public string MassRefreshCronSchedule { get; set; } = "0 0 6 * * SAT";
/// <summary>
/// Cron schedule for daily incremental data refresh. Empty string disables the schedule.
/// </summary>
public string DailyRefreshCronSchedule { get; set; } = "0 0 4 * * *";
/// <summary>
/// Cron schedule for hourly data refresh. Empty string disables the schedule.
/// </summary>
public string HourlyRefreshCronSchedule { get; set; } = "0 0 * * * *";
/// <summary>
/// Maximum number of concurrent update operations.
/// </summary>
public int MaxConcurrentUpdates { get; set; } = 4;
}
@@ -1,27 +0,0 @@
namespace JdeScoping.Core.Options;
/// <summary>
/// Configuration options for Excel export functionality.
/// </summary>
public class ExcelExportOptions
{
/// <summary>
/// Configuration section name in appsettings.json.
/// </summary>
public const string SectionName = "ExcelExport";
/// <summary>
/// Directory for temporary Excel files.
/// </summary>
public string TempDirectory { get; set; } = "/tmp/lotfinder";
/// <summary>
/// Maximum number of rows per Excel sheet.
/// </summary>
public int MaxRowsPerSheet { get; set; } = 1048576;
/// <summary>
/// Default date format for Excel cells.
/// </summary>
public string DefaultDateFormat { get; set; } = "yyyy-MM-dd HH:mm:ss";
}
@@ -1,8 +0,0 @@
namespace JdeScoping.Core.Options;
public class SearchOptions
{
public int MaxResultRows { get; set; } = 100000;
public int TimeoutSeconds { get; set; } = 300;
public int MaxConcurrentSearches { get; set; } = 5;
}
@@ -1,7 +1,6 @@
using JdeScoping.Core.Interfaces;
using JdeScoping.Core.Options;
using JdeScoping.DataAccess;
using JdeScoping.DataAccess.Configuration;
using JdeScoping.DataAccess.Options;
using JdeScoping.DataAccess.FilterHandlers;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataAccess.QueryBuilders;
@@ -1,4 +1,4 @@
namespace JdeScoping.DataAccess.Configuration;
namespace JdeScoping.DataAccess.Options;
/// <summary>
/// Configuration options for the data access layer.
@@ -0,0 +1,27 @@
namespace JdeScoping.DataAccess.Options;
/// <summary>
/// Configuration options for search operations.
/// </summary>
public class SearchOptions
{
/// <summary>
/// Configuration section name in appsettings.json.
/// </summary>
public const string SectionName = "Search";
/// <summary>
/// Maximum number of result rows to return.
/// </summary>
public int MaxResultRows { get; set; } = 100000;
/// <summary>
/// Search query timeout in seconds.
/// </summary>
public int TimeoutSeconds { get; set; } = 300;
/// <summary>
/// Maximum number of concurrent search operations.
/// </summary>
public int MaxConcurrentSearches { get; set; } = 5;
}
@@ -1,4 +1,4 @@
namespace JdeScoping.DataAccess.Configuration;
namespace JdeScoping.DataAccess.Options;
/// <summary>
/// Configuration options for search processing.
@@ -1,4 +1,4 @@
namespace JdeScoping.Core.Options;
namespace JdeScoping.DataAccess.Options;
/// <summary>
/// Configuration options for search processing background service.
@@ -2,7 +2,7 @@ using System.Runtime.CompilerServices;
using Dapper;
using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Quality;
using JdeScoping.DataAccess.Configuration;
using JdeScoping.DataAccess.Options;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataAccess.Queries;
using Microsoft.Extensions.Logging;
@@ -1,6 +1,6 @@
using System.Runtime.CompilerServices;
using Dapper;
using JdeScoping.DataAccess.Configuration;
using JdeScoping.DataAccess.Options;
using JdeScoping.DataAccess.Interfaces;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -1,6 +1,6 @@
using System.Data;
using JdeScoping.Core.Interfaces;
using JdeScoping.DataAccess.Configuration;
using JdeScoping.DataAccess.Options;
using JdeScoping.DataAccess.Exceptions;
using JdeScoping.DataAccess.Interfaces;
using Microsoft.Data.SqlClient;
@@ -1,6 +1,6 @@
using System.Runtime.CompilerServices;
using Dapper;
using JdeScoping.DataAccess.Configuration;
using JdeScoping.DataAccess.Options;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataAccess.Models;
using JdeScoping.DataAccess.Models.Results;
@@ -1,4 +1,4 @@
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Telemetry;
using Microsoft.Extensions.DependencyInjection;
@@ -4,7 +4,7 @@ using JdeScoping.Core.Models.Organization;
using JdeScoping.Core.Models.Quality;
using JdeScoping.Core.Models.WorkOrders;
using JdeScoping.DataSync;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Configuration.MergeConfigurations;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Fetchers.Cms;
@@ -1,5 +1,5 @@
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
namespace JdeScoping.DataSync.Models;
@@ -1,4 +1,4 @@
namespace JdeScoping.DataSync.Configuration;
namespace JdeScoping.DataSync.Options;
/// <summary>
/// Configuration for a single data source table sync.
@@ -1,6 +1,6 @@
using System.ComponentModel.DataAnnotations;
namespace JdeScoping.DataSync.Configuration;
namespace JdeScoping.DataSync.Options;
/// <summary>
/// Configuration options for the data synchronization service.
@@ -1,7 +1,7 @@
using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Infrastructure;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Models;
using Microsoft.Extensions.Logging;
@@ -1,4 +1,4 @@
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Telemetry;
using Microsoft.Extensions.DependencyInjection;
@@ -4,7 +4,7 @@ using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Interfaces;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Models;
using JdeScoping.DataSync.Telemetry;
@@ -1,6 +1,6 @@
using JdeScoping.Core.Interfaces;
using JdeScoping.ExcelIO;
using JdeScoping.ExcelIO.Configuration;
using JdeScoping.ExcelIO.Options;
using JdeScoping.ExcelIO.Generators;
using JdeScoping.ExcelIO.Helpers;
using JdeScoping.ExcelIO.Parsing;
@@ -2,7 +2,7 @@ using System.Reflection;
using ClosedXML.Excel;
using JdeScoping.Core.Interfaces;
using JdeScoping.ExcelIO.Attributes;
using JdeScoping.ExcelIO.Configuration;
using JdeScoping.ExcelIO.Options;
using JdeScoping.ExcelIO.Formatting;
using JdeScoping.ExcelIO.Generators;
using JdeScoping.ExcelIO.Models.Reporting;
@@ -1,5 +1,5 @@
using ClosedXML.Excel;
using JdeScoping.ExcelIO.Configuration;
using JdeScoping.ExcelIO.Options;
using JdeScoping.ExcelIO.Formatting;
using JdeScoping.ExcelIO.Models.Reporting;
using Microsoft.Extensions.Options;
@@ -1,4 +1,4 @@
namespace JdeScoping.ExcelIO.Configuration;
namespace JdeScoping.ExcelIO.Options;
/// <summary>
/// Configuration options for Excel export functionality.
+4 -3
View File
@@ -1,7 +1,9 @@
using JdeScoping.Api;
using JdeScoping.Core.Interfaces;
using JdeScoping.Core.Options;
using JdeScoping.DataAccess.Configuration;
using JdeScoping.DataAccess.Options;
using JdeScoping.DataSync.Options;
using JdeScoping.ExcelIO.Options;
using JdeScoping.Infrastructure.Options;
using JdeScoping.Database;
using Microsoft.Extensions.Options;
@@ -68,7 +70,6 @@ static void ValidateServices(IServiceProvider services)
// Validate Options classes are bound
_ = provider.GetRequiredService<IOptions<DataAccessOptions>>();
_ = provider.GetRequiredService<IOptions<DataSyncOptions>>();
_ = provider.GetRequiredService<IOptions<JdeScoping.DataSync.Configuration.DataSyncOptions>>();
_ = provider.GetRequiredService<IOptions<ExcelExportOptions>>();
_ = provider.GetRequiredService<IOptions<SearchProcessingOptions>>();
_ = provider.GetRequiredService<IOptions<DataSourceOptions>>();
+4 -4
View File
@@ -112,16 +112,16 @@
]
},
"Auth": {
"UseFakeAuth": false,
"CookieName": "ScopingTool.Auth",
"CookieExpirationMinutes": 480,
"AdminBypassUsers": []
"CookieExpirationMinutes": 480
},
"Ldap": {
"ServerUrls": ["ldap.corp.example.com"],
"GroupDn": "CN=ScopingTool-Users,OU=Groups,DC=corp,DC=example,DC=com",
"SearchBase": "DC=corp,DC=example,DC=com",
"ConnectionTimeoutSeconds": 30
"ConnectionTimeoutSeconds": 30,
"UseFakeAuth": false,
"AdminBypassUsers": []
},
"ExcelExport": {
"TempDirectory": "/tmp/lotfinder",
@@ -2,7 +2,7 @@ using System.DirectoryServices.Protocols;
using System.Net;
using JdeScoping.Core.Interfaces;
using JdeScoping.Core.Models;
using JdeScoping.Core.Options;
using JdeScoping.Infrastructure.Options;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -16,16 +16,13 @@ public sealed class LdapAuthService : IAuthService
private const string LdapLookupFormat = "(sAMAccountName={0})";
private readonly LdapOptions _options;
private readonly AuthOptions _authOptions;
private readonly ILogger<LdapAuthService> _logger;
public LdapAuthService(
IOptions<LdapOptions> options,
IOptions<AuthOptions> authOptions,
ILogger<LdapAuthService> logger)
{
_options = options.Value;
_authOptions = authOptions.Value;
_logger = logger;
}
@@ -41,7 +38,7 @@ public sealed class LdapAuthService : IAuthService
}
// Check if user is in admin bypass list
var isAdminBypass = _authOptions.AdminBypassUsers
var isAdminBypass = _options.AdminBypassUsers
.Any(u => string.Equals(u, username, StringComparison.OrdinalIgnoreCase));
// Try each configured LDAP server
@@ -1,4 +1,4 @@
namespace JdeScoping.Core.Options;
namespace JdeScoping.Infrastructure.Options;
/// <summary>
/// Configuration options for data source selection (Oracle vs file-based).
@@ -1,4 +1,4 @@
namespace JdeScoping.Core.Options;
namespace JdeScoping.Infrastructure.Options;
/// <summary>
/// LDAP configuration options for authentication
@@ -32,4 +32,16 @@ public class LdapOptions
/// Connection timeout in seconds.
/// </summary>
public int ConnectionTimeoutSeconds { get; set; } = 30;
/// <summary>
/// Enable fake authentication for development.
/// When true, any credentials are accepted.
/// </summary>
public bool UseFakeAuth { get; set; } = false;
/// <summary>
/// Optional list of usernames that bypass group check.
/// Use sparingly for admin/testing purposes.
/// </summary>
public string[] AdminBypassUsers { get; set; } = [];
}
@@ -3,7 +3,7 @@ using System.Text.Json;
using JdeScoping.Core.Interfaces;
using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Quality;
using JdeScoping.Core.Options;
using JdeScoping.Infrastructure.Options;
using Microsoft.Extensions.Options;
namespace JdeScoping.Infrastructure.Sources.Cms;
@@ -5,7 +5,7 @@ using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Inventory;
using JdeScoping.Core.Models.Organization;
using JdeScoping.Core.Models.WorkOrders;
using JdeScoping.Core.Options;
using JdeScoping.Infrastructure.Options;
using Microsoft.Extensions.Options;
namespace JdeScoping.Infrastructure.Sources.Jde;
@@ -1,4 +1,4 @@
using JdeScoping.DataAccess.Configuration;
using JdeScoping.DataAccess.Options;
using JdeScoping.DataAccess.Exceptions;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataAccess.Repositories;
@@ -24,7 +24,7 @@ public class CmsRepositoryTests
{
_connectionFactory = Substitute.For<IDbConnectionFactory>();
_logger = Substitute.For<ILogger<CmsRepository>>();
_options = Options.Create(new DataAccessOptions
_options = Microsoft.Extensions.Options.Options.Create(new DataAccessOptions
{
DefaultTimeoutSeconds = 30,
MisDataTimeoutSeconds = 60000
@@ -197,7 +197,7 @@ public class CmsRepositoryTests
public void Constructor_UsesMisDataTimeout()
{
// Arrange
var customOptions = Options.Create(new DataAccessOptions
var customOptions = Microsoft.Extensions.Options.Options.Create(new DataAccessOptions
{
MisDataTimeoutSeconds = 999999
});
@@ -214,7 +214,7 @@ public class CmsRepositoryTests
public void Constructor_DefaultMisDataTimeout_Is60000Seconds()
{
// Arrange
var defaultOptions = Options.Create(new DataAccessOptions());
var defaultOptions = Microsoft.Extensions.Options.Options.Create(new DataAccessOptions());
// Act
var repository = new CmsRepository(_connectionFactory, _logger, defaultOptions);
@@ -1,4 +1,4 @@
using JdeScoping.DataAccess.Configuration;
using JdeScoping.DataAccess.Options;
using JdeScoping.DataAccess.Exceptions;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataAccess.Repositories;
@@ -24,7 +24,7 @@ public class JdeRepositoryTests
{
_connectionFactory = Substitute.For<IDbConnectionFactory>();
_logger = Substitute.For<ILogger<JdeRepository>>();
_options = Options.Create(new DataAccessOptions
_options = Microsoft.Extensions.Options.Options.Create(new DataAccessOptions
{
DefaultTimeoutSeconds = 30,
LotUsageTimeoutSeconds = 60,
@@ -637,7 +637,7 @@ public class JdeRepositoryTests
public void Constructor_UsesConfiguredSchemas()
{
// Arrange
var customOptions = Options.Create(new DataAccessOptions
var customOptions = Microsoft.Extensions.Options.Options.Create(new DataAccessOptions
{
ProductionSchema = "CUSTOM_PROD",
ArchiveSchema = "CUSTOM_ARC",
@@ -656,7 +656,7 @@ public class JdeRepositoryTests
public void Constructor_UsesConfiguredTimeouts()
{
// Arrange
var customOptions = Options.Create(new DataAccessOptions
var customOptions = Microsoft.Extensions.Options.Options.Create(new DataAccessOptions
{
DefaultTimeoutSeconds = 120,
LotUsageTimeoutSeconds = 999999
@@ -3,7 +3,7 @@ using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Inventory;
using JdeScoping.Core.Models.Search;
using JdeScoping.Core.ViewModels;
using JdeScoping.DataAccess.Configuration;
using JdeScoping.DataAccess.Options;
using JdeScoping.DataAccess.Exceptions;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataAccess.Repositories;
@@ -29,7 +29,7 @@ public class LotFinderRepositoryTests
{
_connectionFactory = Substitute.For<IDbConnectionFactory>();
_logger = Substitute.For<ILogger<LotFinderRepository>>();
_options = Options.Create(new DataAccessOptions
_options = Microsoft.Extensions.Options.Options.Create(new DataAccessOptions
{
DefaultTimeoutSeconds = 30,
RebuildIndexTimeoutSeconds = 60
@@ -4,7 +4,7 @@ using System.Runtime.CompilerServices;
using JdeScoping.Core.Interfaces;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.IntegrationTests.Infrastructure;
using JdeScoping.DataSync.Models;
@@ -316,7 +316,7 @@ public class TableSyncOperationTests : IAsyncLifetime
return new MassInsertResult(records.Count, TimeSpan.Zero, true);
});
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
BatchSize = 1000,
BulkCopyBatchSize = 100
@@ -1,5 +1,5 @@
using System.Diagnostics.Metrics;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Telemetry;
using Microsoft.Extensions.DependencyInjection;
@@ -22,7 +22,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_WhenDisabled_ExitsImmediately()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = false
});
@@ -56,7 +56,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_WhenEnabled_StartsAndCanBeStopped()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMilliseconds(100)
@@ -106,7 +106,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_GracefulShutdown_CompletesCleanly()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromSeconds(10) // Long interval
@@ -152,7 +152,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_AtStartup_CallsCloseOpenUpdateEntries()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMilliseconds(50)
@@ -199,7 +199,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_WhenCloseOpenEntriesFindsEntries_LogsAndContinues()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMilliseconds(50)
@@ -248,7 +248,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_WhenCloseOpenEntriesThrows_ContinuesStarting()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMilliseconds(50)
@@ -301,7 +301,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_CallsOrchestratorForParallelExecution()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMilliseconds(50),
@@ -349,7 +349,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_WhenOrchestratorThrows_ContinuesNextCycle()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMilliseconds(50)
@@ -404,7 +404,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_WhenCancelled_StopsGracefully()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromSeconds(10)
@@ -462,7 +462,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_PassesCancellationTokenToOrchestrator()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMilliseconds(50)
@@ -510,7 +510,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_WhenCancelledDuringDelay_ExitsCleanly()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMinutes(5) // Long delay
@@ -562,7 +562,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_UsesNewScopePerCycle()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMilliseconds(50)
@@ -610,7 +610,7 @@ public class DataSyncServiceTests
public async Task ExecuteAsync_WhenSyncFails_ContinuesRunning()
{
// Arrange
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
Enabled = true,
CheckInterval = TimeSpan.FromMilliseconds(50)
@@ -1,7 +1,7 @@
using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Infrastructure;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Services;
using Microsoft.Extensions.Logging.Abstractions;
@@ -23,7 +23,7 @@ public class ScheduleCheckerTests
public ScheduleCheckerTests()
{
_repository = Substitute.For<IDataUpdateRepository>();
_options = Options.Create(new DataSyncOptions
_options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
LookbackMultiplier = 3,
DataSources = []
@@ -1,6 +1,6 @@
using System.Diagnostics.Metrics;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Models;
using JdeScoping.DataSync.Services;
@@ -27,7 +27,7 @@ public class SyncOrchestratorTests
public SyncOrchestratorTests()
{
_scheduleChecker = Substitute.For<IScheduleChecker>();
_options = Options.Create(new DataSyncOptions
_options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
MaxDegreeOfParallelism = 4
});
@@ -176,7 +176,7 @@ public class SyncOrchestratorTests
public async Task ExecutePendingSyncsAsync_RespectsMaxDegreeOfParallelism()
{
// Arrange: Create 10 tasks but limit parallelism to 2
var options = Options.Create(new DataSyncOptions
var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
MaxDegreeOfParallelism = 2
});
@@ -5,7 +5,7 @@ using JdeScoping.Core.Interfaces;
using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Models;
using JdeScoping.DataSync.Services;
@@ -40,7 +40,7 @@ public class TableSyncOperationTests
_bulkMergeHelper = Substitute.For<IBulkMergeHelper>();
_configRegistry = Substitute.For<IMergeConfigurationRegistry>();
_options = Options.Create(new DataSyncOptions
_options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
BatchSize = 1000,
BulkCopyBatchSize = 100
@@ -1,5 +1,5 @@
using ClosedXML.Excel;
using JdeScoping.ExcelIO.Configuration;
using JdeScoping.ExcelIO.Options;
using JdeScoping.ExcelIO.Generators;
using JdeScoping.ExcelIO.Helpers;
using JdeScoping.ExcelIO.Models.Reporting;
@@ -16,7 +16,7 @@ public class CriteriaSheetGeneratorTests
public CriteriaSheetGeneratorTests()
{
_options = Options.Create(new ExcelExportOptions
_options = Microsoft.Extensions.Options.Options.Create(new ExcelExportOptions
{
CriteriaSheetPassword = "TestPassword"
});
@@ -1,5 +1,5 @@
using ClosedXML.Excel;
using JdeScoping.ExcelIO.Configuration;
using JdeScoping.ExcelIO.Options;
using JdeScoping.ExcelIO.Generators;
using JdeScoping.ExcelIO.Helpers;
using JdeScoping.ExcelIO.Models.Reporting;
@@ -23,7 +23,7 @@ public class ExcelExportIntegrationTests
public ExcelExportIntegrationTests()
{
_logger = Substitute.For<ILogger<ExcelExportService>>();
_options = Options.Create(new ExcelExportOptions
_options = Microsoft.Extensions.Options.Options.Create(new ExcelExportOptions
{
CriteriaSheetPassword = "TestCriteriaPass",
DataSheetPassword = "TestDataPass"
@@ -1,5 +1,5 @@
using ClosedXML.Excel;
using JdeScoping.ExcelIO.Configuration;
using JdeScoping.ExcelIO.Options;
using JdeScoping.ExcelIO.Generators;
using JdeScoping.ExcelIO.Helpers;
using JdeScoping.ExcelIO.Models.Reporting;
@@ -20,7 +20,7 @@ public class ExcelExportServiceTests
public ExcelExportServiceTests()
{
_logger = Substitute.For<ILogger<ExcelExportService>>();
_options = Options.Create(new ExcelExportOptions
_options = Microsoft.Extensions.Options.Options.Create(new ExcelExportOptions
{
CriteriaSheetPassword = "TestCriteriaPass",
DataSheetPassword = "TestDataPass"
@@ -1,5 +1,5 @@
using ClosedXML.Excel;
using JdeScoping.ExcelIO.Configuration;
using JdeScoping.ExcelIO.Options;
using JdeScoping.ExcelIO.Formatting;
using JdeScoping.ExcelIO.Generators;
using JdeScoping.ExcelIO.Helpers;
@@ -24,7 +24,7 @@ public class LegacyComparisonTests
public LegacyComparisonTests()
{
var logger = Substitute.For<ILogger<ExcelExportService>>();
var options = Options.Create(new ExcelExportOptions
var options = Microsoft.Extensions.Options.Options.Create(new ExcelExportOptions
{
CriteriaSheetPassword = "JDE_SCOPING_TOOL_PASS",
DataSheetPassword = "JDESCOPINGTOOL"
@@ -1,4 +1,4 @@
using JdeScoping.Core.Options;
using JdeScoping.Infrastructure.Options;
using JdeScoping.Infrastructure.Auth;
using JdeScoping.Infrastructure.Tests.Helpers;
using Microsoft.Extensions.Logging;
@@ -52,19 +52,16 @@ public class LdapIntegrationTests
{
// Arrange
var testUser = MockLdapServer.ValidGroupMemberUser;
var ldapOptions = Options.Create(new LdapOptions
var ldapOptions = Microsoft.Extensions.Options.Options.Create(new LdapOptions
{
ServerUrls = MockLdapServer.FakeServerUrls,
GroupDn = MockLdapServer.TestGroupDn,
SearchBase = MockLdapServer.TestSearchBase,
ConnectionTimeoutSeconds = 1 // Fast timeout for test
});
var authOptions = Options.Create(new AuthOptions
{
ConnectionTimeoutSeconds = 1, // Fast timeout for test
AdminBypassUsers = []
});
var service = new LdapAuthService(ldapOptions, authOptions, _logger);
var service = new LdapAuthService(ldapOptions, _logger);
// Act
var result = await service.AuthenticateAsync(testUser.Username, testUser.Password);
@@ -105,19 +102,16 @@ public class LdapIntegrationTests
{
// Arrange
var testUser = MockLdapServer.ValidNotInGroupUser;
var ldapOptions = Options.Create(new LdapOptions
var ldapOptions = Microsoft.Extensions.Options.Options.Create(new LdapOptions
{
ServerUrls = MockLdapServer.FakeServerUrls,
GroupDn = MockLdapServer.TestGroupDn,
SearchBase = MockLdapServer.TestSearchBase,
ConnectionTimeoutSeconds = 1
});
var authOptions = Options.Create(new AuthOptions
{
ConnectionTimeoutSeconds = 1,
AdminBypassUsers = []
});
var service = new LdapAuthService(ldapOptions, authOptions, _logger);
var service = new LdapAuthService(ldapOptions, _logger);
// Act
var result = await service.AuthenticateAsync(testUser.Username, testUser.Password);
@@ -152,19 +146,16 @@ public class LdapIntegrationTests
{
// Arrange
var testUser = MockLdapServer.InvalidCredentialsUser;
var ldapOptions = Options.Create(new LdapOptions
var ldapOptions = Microsoft.Extensions.Options.Options.Create(new LdapOptions
{
ServerUrls = MockLdapServer.FakeServerUrls,
GroupDn = MockLdapServer.TestGroupDn,
SearchBase = MockLdapServer.TestSearchBase,
ConnectionTimeoutSeconds = 1
});
var authOptions = Options.Create(new AuthOptions
{
ConnectionTimeoutSeconds = 1,
AdminBypassUsers = []
});
var service = new LdapAuthService(ldapOptions, authOptions, _logger);
var service = new LdapAuthService(ldapOptions, _logger);
// Act
var result = await service.AuthenticateAsync(testUser.Username, testUser.Password);
@@ -205,19 +196,16 @@ public class LdapIntegrationTests
public async Task AuthenticateAsync_AllServersFail_ReturnsConnectionError()
{
// Arrange
var ldapOptions = Options.Create(new LdapOptions
var ldapOptions = Microsoft.Extensions.Options.Options.Create(new LdapOptions
{
ServerUrls = MockLdapServer.FakeServerUrls, // 3 fake servers that will all fail
GroupDn = MockLdapServer.TestGroupDn,
SearchBase = MockLdapServer.TestSearchBase,
ConnectionTimeoutSeconds = 1 // Fast timeout for test
});
var authOptions = Options.Create(new AuthOptions
{
ConnectionTimeoutSeconds = 1, // Fast timeout for test
AdminBypassUsers = []
});
var service = new LdapAuthService(ldapOptions, authOptions, _logger);
var service = new LdapAuthService(ldapOptions, _logger);
// Act
var result = await service.AuthenticateAsync("anyuser", "anypassword");
@@ -1,5 +1,5 @@
using JdeScoping.Core.Options;
using JdeScoping.Infrastructure.Auth;
using JdeScoping.Infrastructure.Options;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using NSubstitute;
@@ -10,20 +10,16 @@ namespace JdeScoping.Infrastructure.Tests.Unit;
public class LdapAuthServiceTests
{
private readonly IOptions<LdapOptions> _ldapOptions;
private readonly IOptions<AuthOptions> _authOptions;
private readonly ILogger<LdapAuthService> _logger;
public LdapAuthServiceTests()
{
_ldapOptions = Options.Create(new LdapOptions
_ldapOptions = Microsoft.Extensions.Options.Options.Create(new LdapOptions
{
ServerUrls = ["ldap.test.com"],
GroupDn = "CN=TestGroup,DC=test,DC=com",
SearchBase = "DC=test,DC=com",
ConnectionTimeoutSeconds = 5
});
_authOptions = Options.Create(new AuthOptions
{
ConnectionTimeoutSeconds = 5,
AdminBypassUsers = []
});
_logger = Substitute.For<ILogger<LdapAuthService>>();
@@ -33,7 +29,7 @@ public class LdapAuthServiceTests
public async Task AuthenticateAsync_EmptyUsername_ReturnsFailure()
{
// Arrange
var service = new LdapAuthService(_ldapOptions, _authOptions, _logger);
var service = new LdapAuthService(_ldapOptions, _logger);
// Act
var result = await service.AuthenticateAsync("", "password");
@@ -47,7 +43,7 @@ public class LdapAuthServiceTests
public async Task AuthenticateAsync_EmptyPassword_ReturnsFailure()
{
// Arrange
var service = new LdapAuthService(_ldapOptions, _authOptions, _logger);
var service = new LdapAuthService(_ldapOptions, _logger);
// Act
var result = await service.AuthenticateAsync("user", "");
@@ -61,13 +57,14 @@ public class LdapAuthServiceTests
public async Task AuthenticateAsync_NoServersConfigured_ReturnsConnectionError()
{
// Arrange
var emptyServerOptions = Options.Create(new LdapOptions
var emptyServerOptions = Microsoft.Extensions.Options.Options.Create(new LdapOptions
{
ServerUrls = [],
GroupDn = "CN=TestGroup,DC=test,DC=com",
SearchBase = "DC=test,DC=com"
SearchBase = "DC=test,DC=com",
AdminBypassUsers = []
});
var service = new LdapAuthService(emptyServerOptions, _authOptions, _logger);
var service = new LdapAuthService(emptyServerOptions, _logger);
// Act
var result = await service.AuthenticateAsync("user", "password");
@@ -81,7 +78,7 @@ public class LdapAuthServiceTests
public void GetUserInfoAsync_ThrowsNotSupportedException()
{
// Arrange
var service = new LdapAuthService(_ldapOptions, _authOptions, _logger);
var service = new LdapAuthService(_ldapOptions, _logger);
// Act & Assert
Should.Throw<NotSupportedException>(() => service.GetUserInfoAsync("user").GetAwaiter().GetResult());
@@ -93,11 +90,15 @@ public class LdapAuthServiceTests
// Arrange
// Note: We can't fully test admin bypass without a real LDAP server since bind still happens.
// This test verifies the configuration is recognized by checking that bypass users are configured.
var authOptionsWithBypass = Options.Create(new AuthOptions
var ldapOptionsWithBypass = Microsoft.Extensions.Options.Options.Create(new LdapOptions
{
ServerUrls = ["ldap.test.com"],
GroupDn = "CN=TestGroup,DC=test,DC=com",
SearchBase = "DC=test,DC=com",
ConnectionTimeoutSeconds = 5,
AdminBypassUsers = ["bypassuser", "adminuser"]
});
var service = new LdapAuthService(_ldapOptions, authOptionsWithBypass, _logger);
var service = new LdapAuthService(ldapOptionsWithBypass, _logger);
// Act - attempt to authenticate the bypass user (will fail LDAP connection, but config is exercised)
var result = await service.AuthenticateAsync("bypassuser", "anypassword");
@@ -113,14 +114,15 @@ public class LdapAuthServiceTests
public async Task AuthenticateAsync_MultipleServersConfigured_TriesEachUntilAllFail()
{
// Arrange
var multiServerOptions = Options.Create(new LdapOptions
var multiServerOptions = Microsoft.Extensions.Options.Options.Create(new LdapOptions
{
ServerUrls = ["ldap1.test.com", "ldap2.test.com", "ldap3.test.com"],
GroupDn = "CN=TestGroup,DC=test,DC=com",
SearchBase = "DC=test,DC=com",
ConnectionTimeoutSeconds = 1 // Fast timeout for test
ConnectionTimeoutSeconds = 1, // Fast timeout for test
AdminBypassUsers = []
});
var service = new LdapAuthService(multiServerOptions, _authOptions, _logger);
var service = new LdapAuthService(multiServerOptions, _logger);
// Act
var result = await service.AuthenticateAsync("testuser", "testpassword");
+134
View File
@@ -0,0 +1,134 @@
# Encrypted Login Design
## Overview
Consolidate login models into Core project and implement RSA-OAEP encryption for login credentials between Blazor WASM client and API.
## Decisions
| Decision | Choice | Rationale |
|----------|--------|-----------|
| Encryption algorithm | RSA-OAEP (SHA-256) | Simple direct encryption, suitable for small payloads |
| Key size | 2048-bit | Industry standard, ~20 year security horizon |
| Key distribution | Fetch at runtime | Allows key rotation without client redeployment |
| Key storage | Auto-generate, persist to file | Self-bootstrapping, no manual key management |
| Key expiration | None | Internal enterprise app, manual rotation if needed |
## New Models (Core)
All in `JdeScoping.Core/Models/Auth/`:
```csharp
// LoginModel.cs - shared form/request model
public class LoginModel
{
[Required(ErrorMessage = "Username is required")]
public string Username { get; set; } = string.Empty;
[Required(ErrorMessage = "Password is required")]
public string Password { get; set; } = string.Empty;
}
// LoginResultModel.cs - API response (unencrypted)
public record LoginResultModel(
bool Success,
string? ErrorMessage,
UserInfo? User);
// EncryptedLoginRequest.cs - encrypted payload wrapper
public record EncryptedLoginRequest(string EncryptedData);
// PublicKeyResponse.cs - public key endpoint response
public record PublicKeyResponse(string PublicKeyPem);
```
## RSA Key Service (Infrastructure)
**Interface:** `JdeScoping.Core/Interfaces/IRsaKeyService.cs`
```csharp
public interface IRsaKeyService
{
string GetPublicKeyPem();
byte[] Decrypt(byte[] ciphertext);
}
```
**Implementation:** `JdeScoping.Infrastructure/Security/RsaKeyService.cs`
- Auto-generates 2048-bit RSA key on first startup if not exists
- Persists private key to configurable file path
- Exports public key as PEM format
- Decrypts using RSA-OAEP with SHA-256 padding
## API Changes
**New endpoint:**
```
GET /api/auth/public-key → PublicKeyResponse
```
**Modified endpoint:**
```
POST /api/auth/login
Body: EncryptedLoginRequest { EncryptedData: base64 }
Response: LoginResultModel
```
Flow:
1. Receive base64-encoded encrypted data
2. Decode and decrypt with RSA private key
3. Deserialize JSON to LoginModel
4. Authenticate via IAuthService
5. Return LoginResultModel (unencrypted)
## Client Changes
**New service:** `ICryptoService` / `CryptoService`
- Uses `Blazor.SubtleCrypto` NuGet package
- Fetches and caches server public key
- Encrypts LoginModel JSON with RSA-OAEP
- Returns base64-encoded ciphertext
**Modified AuthService:**
- Injects ICryptoService
- Encrypts LoginModel before sending
- Sends EncryptedLoginRequest to API
## Files to Delete
- `JdeScoping.Api/Models/LoginRequest.cs`
- `JdeScoping.Client/Models/LoginModel.cs`
## Dependencies
**JdeScoping.Client.csproj:**
```xml
<PackageReference Include="Blazor.SubtleCrypto" Version="..." />
```
## DI Registration
**API/Host:**
```csharp
services.AddSingleton<IRsaKeyService>(sp =>
new RsaKeyService(Path.Combine(appDataPath, "rsa-key.bin")));
```
**Client:**
```csharp
services.AddScoped<ICryptoService, CryptoService>();
```
## Test Coverage
### RsaKeyServiceTests (Api.Tests)
- GetPublicKeyPem returns valid PEM format
- Decrypt with valid ciphertext returns plaintext
- Constructor loads existing key from file
- Constructor generates and persists new key if missing
### CryptoServiceTests (Client.Tests)
- EncryptLoginAsync returns valid base64
- EncryptLoginAsync caches public key (single HTTP call)
### Integration (optional)
- Full roundtrip: encrypt on client → decrypt on API → authenticate
File diff suppressed because it is too large Load Diff
+334
View File
@@ -0,0 +1,334 @@
# ETL Pipeline Design for DataSync
**Date:** 2026-01-03
**Status:** Reviewed (Codex MCP)
**Purpose:** Replace the existing strongly-typed fetcher + source-generated DataReader + BulkMergeHelper system with a flexible, configuration-driven ETL pipeline.
## Problem Statement
The current DataSync system requires:
1. A source-generated `IDataReader` implementation per entity type
2. Strongly-typed `IDataFetcher<T>` classes for each data source
3. Code changes to modify queries or add new sync operations
**Primary drivers for change:**
- **Flexibility** - Enable ad-hoc queries and schema changes without code modifications
- **Observability** - Better error handling and row count tracking across pipeline steps
## Design Overview
```
┌─────────────┐ ┌──────────────┐ ┌─────────────┐
│ ImportSource│───▶│ Transformer │───▶│ Destination │
│ (IDataReader) │ (decorates) │ │ (bulk ops) │
└─────────────┘ └──────────────┘ └─────────────┘
▲ │
│ ▼
┌──────┴──────┐ ┌──────────────┐
│ Pre-Scripts │ │ Post-Scripts │
└─────────────┘ └──────────────┘
```
All steps report to `PipelineResult` with timing and row counts.
## Core Interfaces
### IImportSource
Reads data from a source system, returns `IDataReader` for streaming.
```csharp
public interface IImportSource : IAsyncDisposable
{
Task<IDataReader> ReadDataAsync(CancellationToken cancellationToken = default);
string SourceName { get; }
}
```
**Implementations:**
- `DbQuerySource` - Executes SQL against Oracle/Sybase/SQL Server via `IDbConnectionFactory`
- `FileSource` - Reads CSV files with configurable delimiter/header options
### IDataTransformer
Wraps an `IDataReader` and applies transformations (column rename, drop, type conversion).
```csharp
public interface IDataTransformer
{
IDataReader Transform(IDataReader source);
string TransformerName { get; }
}
```
**Implementations:**
- `JdeDateTransformer` - Merges Julian date + time columns into `DateTime`
- `ColumnRenameTransformer` - Renames columns
- `ColumnDropTransformer` - Removes columns from output
- `ColumnAddTransformer` - Adds computed columns
Transformers are registered by name in `ITransformerRegistry` for config-based hydration.
### IImportDestination
Writes data to a target table using bulk operations.
```csharp
public interface IImportDestination
{
Task<DestinationResult> WriteAsync(
IDataReader source,
CancellationToken cancellationToken = default);
string DestinationName { get; }
}
public record DestinationResult(
long RowsProcessed,
int BatchCount,
TimeSpan Elapsed);
```
**Implementations:**
#### DbBulkImportDestination (Full Refresh)
1. Truncate destination table
2. Bulk copy all data in batches
3. Per-batch commits
#### DbBulkMergeDestination (Incremental)
1. Create temp table from destination schema
2. For each batch:
- Bulk copy to temp table
- Execute MERGE to destination
- Truncate temp table
3. Drop temp table on completion
**Configuration options:**
- `tableName` - Target table
- `matchColumns` - PK columns for MERGE ON clause
- `updateColumns` - Columns to update (default: all non-match columns)
- `batchSize` - Rows per batch (default: 10000)
### IScriptRunner
Executes SQL scripts before or after the main pipeline.
```csharp
public interface IScriptRunner
{
Task ExecuteAsync(CancellationToken cancellationToken = default);
string ScriptName { get; }
}
```
**Common scripts (factory methods):**
- `CommonScripts.DisableIndexes(factory, tableName)`
- `CommonScripts.RebuildIndexes(factory, tableName)`
- `CommonScripts.UpdateStatistics(factory, tableName)`
- `CommonScripts.CustomSql(factory, sql, name)`
### Pipeline Results
```csharp
public record PipelineResult(
bool Success,
long TotalRows,
TimeSpan Elapsed,
IReadOnlyList<StepResult> Steps,
Exception? Error = null);
public record StepResult(
string StepName,
string StepType, // "Source", "Transform", "Destination", "Script"
long RowsAffected,
TimeSpan Elapsed);
```
## EtlPipeline Orchestration
```csharp
public class EtlPipeline
{
public string PipelineName { get; }
public async Task<PipelineResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
// 1. Run pre-scripts (fail-fast)
// 2. Open source, get IDataReader
// 3. Chain transformers (decorators)
// 4. Write to destination
// 5. Run post-scripts
// 6. Return PipelineResult with all step metrics
}
}
```
**Error handling:**
- Fail-fast on any error
- No cross-batch transactions (per-batch commits)
- Caller marks overall sync operation as failed
- Restart from beginning on failure (no resume)
## Pipeline Construction
### Fluent Builder
```csharp
var pipeline = new EtlPipelineBuilder()
.WithName("WorkOrderSync")
.WithSource(new DbQuerySource(connFactory, "JDE", workOrderQuery))
.WithTransformer(new JdeDateTransformer("UPMJ", "TDAY", "UpdatedAt"))
.WithDestination(new DbBulkMergeDestination(connFactory, "WorkOrder", ["OrderNumber"]))
.WithPreScript(CommonScripts.DisableIndexes(connFactory, "WorkOrder"))
.WithPostScript(CommonScripts.RebuildIndexes(connFactory, "WorkOrder"))
.WithLogger(logger)
.Build();
var result = await pipeline.ExecuteAsync(cancellationToken);
```
### Configuration-Based
```json
{
"name": "WorkOrderSync",
"source": {
"type": "db",
"connectionName": "JDE",
"query": "SELECT * FROM F4801 WHERE UPMJ >= :minDate"
},
"transformers": [
{ "type": "jde-date", "options": { "dateColumn": "UPMJ", "timeColumn": "TDAY", "outputColumn": "UpdatedAt" } }
],
"destination": {
"type": "bulk-merge",
"tableName": "WorkOrder",
"matchColumns": ["OrderNumber"],
"batchSize": 10000
},
"preScripts": [
{ "type": "disable-indexes", "tableName": "WorkOrder" }
],
"postScripts": [
{ "type": "rebuild-indexes", "tableName": "WorkOrder" }
]
}
```
Hydrated via `EtlPipelineFactory.CreateFromConfig(config)`.
## Project Structure
```
NEW/src/JdeScoping.DataSync/
├── Contracts/
│ ├── IImportSource.cs
│ ├── IDataTransformer.cs
│ ├── IImportDestination.cs
│ ├── IScriptRunner.cs
│ └── ITransformerRegistry.cs
├── Etl/
│ ├── Pipeline/
│ │ ├── EtlPipeline.cs
│ │ ├── EtlPipelineBuilder.cs
│ │ └── EtlPipelineFactory.cs
│ ├── Sources/
│ │ ├── DbQuerySource.cs
│ │ └── FileSource.cs
│ ├── Transformers/
│ │ ├── DataTransformerBase.cs
│ │ ├── TransformingDataReader.cs
│ │ ├── JdeDateTransformer.cs
│ │ ├── ColumnRenameTransformer.cs
│ │ ├── ColumnDropTransformer.cs
│ │ └── TransformerRegistry.cs
│ ├── Destinations/
│ │ ├── DbBulkImportDestination.cs
│ │ └── DbBulkMergeDestination.cs
│ ├── Scripts/
│ │ ├── SqlScriptRunner.cs
│ │ └── CommonScripts.cs
│ └── Results/
│ ├── PipelineResult.cs
│ ├── StepResult.cs
│ └── DestinationResult.cs
└── Config/
└── PipelineConfig.cs
```
## Integration Strategy
1. **Parallel existence** - New ETL pipeline lives alongside existing `IDataFetcher<T>` + `IBulkMergeHelper`
2. **Gradual migration** - Convert one sync operation at a time to new pipeline
3. **Shared infrastructure** - Both use `IDbConnectionFactory`, logging, etc.
4. **Deprecation path** - Once all syncs migrated, remove old `Fetchers/` and source generator
## DI Registration
```csharp
public static class EtlServiceCollectionExtensions
{
public static IServiceCollection AddEtlPipeline(this IServiceCollection services)
{
services.AddSingleton<ITransformerRegistry, TransformerRegistry>();
services.AddTransient<EtlPipelineFactory>();
return services;
}
}
```
## Design Decisions
Based on Codex MCP review, the following decisions were made:
### Atomicity & Consistency
| Concern | Decision | Rationale |
|---------|----------|-----------|
| Full refresh partial visibility | **Accept** | Caller retries on failure; consumers tolerate brief inconsistency |
| Orphaned disabled indexes | **Accept risk** | Next successful run rebuilds; manual intervention rare |
| Temp table connection lifetime | **Destination owns connection** | `DbBulkMergeDestination` holds single connection for all batches |
### Implementation Requirements
1. **TransformingDataReader schema accuracy** - Transformers MUST correctly implement:
- `GetSchemaTable()` - Accurate after column changes
- `GetName(ordinal)` - Reflects renames
- `GetOrdinal(name)` - Works with new names
- `GetFieldType(ordinal)` - Correct after type transforms
- `FieldCount` - Accurate after drops/adds
2. **Transform step row counts** - `StepResult.RowsAffected` is `0` for transform steps (inline decorators). Only destination reports actual rows processed.
3. **Cancellation handling** - Destinations use `DbDataReader.ReadAsync()` internally, checking cancellation between rows. Source `IDataReader` interface stays synchronous for ADO.NET compatibility.
4. **Schema validation** - Optional `ValidateSchema` step available to compare source columns against expected schema before pipeline runs. Not mandatory.
5. **MERGE duplicate keys** - Source data MUST have unique keys for match columns. Pipeline does not dedupe; caller's responsibility to ensure uniqueness.
### Connection Lifetime Rules
```
DbQuerySource:
- Opens connection in ReadDataAsync()
- Holds connection until DisposeAsync()
- Connection closed when pipeline completes or fails
DbBulkImportDestination:
- Opens connection in WriteAsync()
- Holds for all batches
- Closes on completion or failure
DbBulkMergeDestination:
- Opens connection in WriteAsync()
- Creates #temp table
- Holds same connection for all batch cycles
- Drops temp table in finally block
- Closes connection on completion or failure
```
## Open Questions
1. Should `DbQuerySource` support parameterized queries with different parameter styles (`:param` for Oracle, `@param` for SQL Server)?
2. Should we support multiple destinations per pipeline (e.g., write to both current and history tables)?
3. How should pipeline configurations be stored and loaded (appsettings.json, separate files, database)?
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,116 @@
# Options Classes Relocation Design
## Summary
Move options classes from Core project to the projects that use them, establishing an `Options/` folder convention across all projects.
## Current State
All 7 options classes live in `JdeScoping.Core/Options/`:
- `AuthOptions` - used by Api + Infrastructure
- `DataSourceOptions` - used by Infrastructure
- `DataSyncOptions` - duplicate (real one in DataSync)
- `ExcelExportOptions` - duplicate (real one in ExcelIO)
- `LdapOptions` - used by Infrastructure
- `SearchOptions` - unused (future use in DataAccess)
- `SearchProcessingOptions` - used by DataAccess
## Target State
### Folder Structure
```
src/
├── JdeScoping.Api/
│ └── Options/
│ └── AuthOptions.cs
├── JdeScoping.DataAccess/
│ └── Options/
│ ├── SearchOptions.cs
│ └── SearchProcessingOptions.cs
├── JdeScoping.Infrastructure/
│ └── Options/
│ ├── DataSourceOptions.cs
│ └── LdapOptions.cs
├── JdeScoping.DataSync/
│ └── Options/ # Renamed from Configuration/
│ └── DataSyncOptions.cs
├── JdeScoping.ExcelIO/
│ └── Options/ # Renamed from Configuration/
│ └── ExcelExportOptions.cs
└── JdeScoping.Core/
└── (Options folder deleted)
```
### Property Moves
**LdapOptions** gains properties from AuthOptions:
- `UseFakeAuth` (bool, default: false)
- `AdminBypassUsers` (string[], default: [])
**AuthOptions** loses those properties, keeping only:
- `CookieName` (string, default: "ScopingTool.Auth")
- `CookieExpirationMinutes` (int, default: 480)
### Namespace Changes
| Old Namespace | New Namespace |
|---------------|---------------|
| `JdeScoping.Core.Options.AuthOptions` | `JdeScoping.Api.Options.AuthOptions` |
| `JdeScoping.Core.Options.SearchOptions` | `JdeScoping.DataAccess.Options.SearchOptions` |
| `JdeScoping.Core.Options.SearchProcessingOptions` | `JdeScoping.DataAccess.Options.SearchProcessingOptions` |
| `JdeScoping.Core.Options.DataSourceOptions` | `JdeScoping.Infrastructure.Options.DataSourceOptions` |
| `JdeScoping.Core.Options.LdapOptions` | `JdeScoping.Infrastructure.Options.LdapOptions` |
| `JdeScoping.DataSync.Configuration.*` | `JdeScoping.DataSync.Options.*` |
| `JdeScoping.ExcelIO.Configuration.*` | `JdeScoping.ExcelIO.Options.*` |
### DI Registration Changes
**Infrastructure/DependencyInjection.cs:**
- Remove `AuthOptions` registration
- Change `authOptions?.UseFakeAuth` check to use `ldapOptions?.UseFakeAuth`
**Api/DependencyInjection.cs:**
- Add `AuthOptions` registration
### Configuration (appsettings.json)
Move settings from `"Auth"` to `"Ldap"` section:
```json
{
"Ldap": {
"ServerUrls": ["ldap.corp.example.com"],
"GroupDn": "CN=ScopingTool-Users,OU=Groups,DC=corp,DC=example,DC=com",
"SearchBase": "DC=corp,DC=example,DC=com",
"ConnectionTimeoutSeconds": 30,
"UseFakeAuth": false,
"AdminBypassUsers": []
},
"Auth": {
"CookieName": "ScopingTool.Auth",
"CookieExpirationMinutes": 480
}
}
```
## Files to Delete
- `src/JdeScoping.Core/Options/AuthOptions.cs`
- `src/JdeScoping.Core/Options/DataSourceOptions.cs`
- `src/JdeScoping.Core/Options/DataSyncOptions.cs`
- `src/JdeScoping.Core/Options/ExcelExportOptions.cs`
- `src/JdeScoping.Core/Options/LdapOptions.cs`
- `src/JdeScoping.Core/Options/SearchOptions.cs`
- `src/JdeScoping.Core/Options/SearchProcessingOptions.cs`
- `src/JdeScoping.Core/Options/` (folder)
## Files to Update
- `Host/Program.cs` - update using statements
- `Infrastructure/DependencyInjection.cs` - remove AuthOptions, use LdapOptions for UseFakeAuth
- `Infrastructure/Auth/LdapAuthService.cs` - update using, use LdapOptions for AdminBypassUsers
- `Api/DependencyInjection.cs` - add AuthOptions registration
- `DataAccess/DependencyInjection.cs` - update using statements
- `DataSync/**` - rename Configuration namespace to Options
- `ExcelIO/**` - rename Configuration namespace to Options
- `appsettings.json` - move UseFakeAuth/AdminBypassUsers to Ldap section