Initial commit: JDE Scoping Tool migration project
Set up repository with legacy .NET Framework 4.8 source (OLD/), new .NET 10 Blazor solution (NEW/), OpenSpec specifications, documentation, and project configuration.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,376 @@
|
||||
# Data Access Design
|
||||
|
||||
## Overview
|
||||
|
||||
This document describes the architecture and implementation approach for the data access layer, including repository interfaces, connection factory, exception handling, and service registration.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Repository Pattern
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ Service Layer │
|
||||
│ (SearchProcessor, DataSyncService, etc.) │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
┌─────────────┼─────────────┐
|
||||
▼ ▼ ▼
|
||||
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
||||
│ILotFinder │ │IJde │ │ICms │
|
||||
│Repository │ │Repository │ │Repository │
|
||||
└──────┬───────┘ └──────┬───────┘ └──────┬───────┘
|
||||
│ │ │
|
||||
┌──────▼───────┐ ┌──────▼───────┐ ┌──────▼───────┐
|
||||
│LotFinder │ │Jde │ │Cms │
|
||||
│Repository │ │Repository │ │Repository │
|
||||
└──────┬───────┘ └──────┬───────┘ └──────┬───────┘
|
||||
│ │ │
|
||||
└────────────────┼────────────────┘
|
||||
▼
|
||||
┌─────────────────────────┐
|
||||
│ IDbConnectionFactory │
|
||||
└─────────────┬───────────┘
|
||||
│
|
||||
┌─────────────────────┼─────────────────────┐
|
||||
▼ ▼ ▼
|
||||
┌───────────────┐ ┌───────────────┐ ┌───────────────┐
|
||||
│ SqlConnection │ │OracleConnection│ │OracleConnection│
|
||||
│ (LotFinderDB) │ │ (JDE/Stage) │ │ (CMS) │
|
||||
└───────────────┘ └───────────────┘ └───────────────┘
|
||||
```
|
||||
|
||||
### Project Structure
|
||||
|
||||
```
|
||||
NEW/src/JdeScoping.DataAccess/
|
||||
├── Exceptions/
|
||||
│ ├── DataAccessException.cs
|
||||
│ ├── ConnectionException.cs
|
||||
│ ├── QueryException.cs
|
||||
│ └── DataAccessTimeoutException.cs
|
||||
├── Interfaces/
|
||||
│ ├── IDbConnectionFactory.cs
|
||||
│ ├── ILotFinderRepository.cs
|
||||
│ ├── IJdeRepository.cs
|
||||
│ └── ICmsRepository.cs
|
||||
├── Repositories/
|
||||
│ ├── LotFinderRepository.cs
|
||||
│ ├── JdeRepository.cs
|
||||
│ └── CmsRepository.cs
|
||||
├── Queries/
|
||||
│ ├── LotFinderQueries.cs (const string SQL statements)
|
||||
│ ├── JdeQueries.cs (const string SQL statements)
|
||||
│ └── CmsQueries.cs (const string SQL statements)
|
||||
├── Configuration/
|
||||
│ └── DataAccessOptions.cs
|
||||
├── DbConnectionFactory.cs
|
||||
├── ServiceCollectionExtensions.cs
|
||||
└── JdeScoping.DataAccess.csproj
|
||||
```
|
||||
|
||||
## Connection Factory
|
||||
|
||||
### IDbConnectionFactory Interface
|
||||
|
||||
```csharp
|
||||
public interface IDbConnectionFactory
|
||||
{
|
||||
Task<SqlConnection> CreateLotFinderConnectionAsync(CancellationToken ct = default);
|
||||
Task<OracleConnection> CreateJdeConnectionAsync(CancellationToken ct = default);
|
||||
Task<OracleConnection> CreateJdeStageConnectionAsync(CancellationToken ct = default);
|
||||
Task<OracleConnection> CreateCmsConnectionAsync(CancellationToken ct = default);
|
||||
}
|
||||
```
|
||||
|
||||
### Implementation Details
|
||||
|
||||
- Registered as **singleton** (stateless, creates new connections)
|
||||
- Connection strings read from `IConfiguration["ConnectionStrings:*"]`
|
||||
- Secrets retrieved from .NET Secret Manager (local) or Azure Key Vault (production)
|
||||
- Connections opened asynchronously before returning
|
||||
- Caller responsible for disposing returned connections
|
||||
|
||||
### Connection String Keys
|
||||
|
||||
| Key | Database | Driver |
|
||||
|-----|----------|--------|
|
||||
| `ConnectionStrings:LotFinderDB` | SQL Server cache | Microsoft.Data.SqlClient |
|
||||
| `ConnectionStrings:JDE` | JDE Oracle (PRODDTA) | Oracle.ManagedDataAccess.Core |
|
||||
| `ConnectionStrings:JDEStage` | JDE Oracle (JDESTAGE) | Oracle.ManagedDataAccess.Core |
|
||||
| `ConnectionStrings:CMS` | CMS Oracle (INFODBA) | Oracle.ManagedDataAccess.Core |
|
||||
|
||||
## Repository Interfaces
|
||||
|
||||
### Registration Lifetimes
|
||||
|
||||
| Interface | Lifetime | Rationale |
|
||||
|-----------|----------|-----------|
|
||||
| `IDbConnectionFactory` | Singleton | Stateless, creates new connections |
|
||||
| `ILotFinderRepository` | Scoped | Per-request, uses scoped DbContext pattern |
|
||||
| `IJdeRepository` | Scoped | Per-request, creates connections as needed |
|
||||
| `ICmsRepository` | Scoped | Per-request, creates connections as needed |
|
||||
|
||||
### Constructor Dependencies
|
||||
|
||||
All repository implementations receive:
|
||||
- `IDbConnectionFactory` - For database connections
|
||||
- `ILogger<T>` - For structured logging
|
||||
- `IOptions<DataAccessOptions>` - For configurable timeouts and schemas
|
||||
|
||||
## Async Streaming Pattern
|
||||
|
||||
### IAsyncEnumerable for Large Datasets
|
||||
|
||||
JDE and CMS repositories return `IAsyncEnumerable<T>` for all collection queries:
|
||||
|
||||
```csharp
|
||||
public async IAsyncEnumerable<WorkOrder> GetWorkOrdersAsync(
|
||||
DateTime? lastUpdateDT = null,
|
||||
[EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
await using var connection = await _connectionFactory.CreateJdeConnectionAsync(ct);
|
||||
|
||||
var sql = lastUpdateDT.HasValue
|
||||
? JdeQueries.SQL_GET_WORKORDERS_FILTERED
|
||||
: JdeQueries.SQL_GET_WORKORDERS;
|
||||
|
||||
var parameters = BuildWorkOrderParameters(lastUpdateDT);
|
||||
|
||||
await foreach (var workOrder in connection.QueryUnbufferedAsync<WorkOrder>(
|
||||
sql, parameters, commandTimeout: _options.Value.DefaultTimeoutSeconds)
|
||||
.WithCancellation(ct))
|
||||
{
|
||||
yield return workOrder;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Benefits
|
||||
|
||||
- Memory efficient: rows streamed one at a time
|
||||
- Cancellation support: stops iteration on cancellation
|
||||
- Backpressure: consumer controls iteration speed
|
||||
- Compatible with `await foreach` syntax
|
||||
|
||||
## Query Management
|
||||
|
||||
### SQL Query Storage
|
||||
|
||||
SQL queries stored as compile-time constants in static classes:
|
||||
|
||||
```csharp
|
||||
public static class JdeQueries
|
||||
{
|
||||
public const string SQL_GET_WORKORDERS = @"
|
||||
SELECT wo.WADOCO AS WorkOrderNumber,
|
||||
TRIM(wo.WAMMCU) AS BranchCode,
|
||||
-- ... rest of query
|
||||
FROM {ProductionSchema}.F4801 wo";
|
||||
|
||||
public const string SQL_GET_WORKORDERS_FILTERED = SQL_GET_WORKORDERS + @"
|
||||
WHERE (wo.WAUPMJ > :dateUpdated OR
|
||||
(wo.WAUPMJ = :dateUpdated AND wo.WATDAY >= :timeUpdated))";
|
||||
}
|
||||
```
|
||||
|
||||
### Schema Placeholder Replacement
|
||||
|
||||
Schema names replaced at runtime from `DataAccessOptions`:
|
||||
|
||||
```csharp
|
||||
private string ApplySchemaPlaceholders(string sql)
|
||||
{
|
||||
return sql
|
||||
.Replace("{ProductionSchema}", _options.Value.ProductionSchema)
|
||||
.Replace("{ArchiveSchema}", _options.Value.ArchiveSchema)
|
||||
.Replace("{StageSchema}", _options.Value.StageSchema);
|
||||
}
|
||||
```
|
||||
|
||||
## Exception Handling
|
||||
|
||||
### Exception Hierarchy
|
||||
|
||||
```
|
||||
Exception
|
||||
└── DataAccessException (base for all data access errors)
|
||||
├── ConnectionException (connection failures)
|
||||
├── QueryException (query execution failures)
|
||||
└── DataAccessTimeoutException (timeout errors)
|
||||
```
|
||||
|
||||
### Exception Properties
|
||||
|
||||
```csharp
|
||||
public class DataAccessException : Exception
|
||||
{
|
||||
public string? Operation { get; } // Method name (e.g., "GetWorkOrdersAsync")
|
||||
public string? Repository { get; } // Repository name (e.g., "JdeRepository")
|
||||
}
|
||||
|
||||
public class ConnectionException : DataAccessException
|
||||
{
|
||||
public string? DataSource { get; } // Database identifier (e.g., "JDE", "CMS")
|
||||
}
|
||||
|
||||
public class QueryException : DataAccessException
|
||||
{
|
||||
public string? QueryName { get; } // Query identifier (e.g., "SQL_GET_WORKORDERS")
|
||||
}
|
||||
|
||||
public class DataAccessTimeoutException : DataAccessException
|
||||
{
|
||||
public int TimeoutSeconds { get; } // Configured timeout value
|
||||
}
|
||||
```
|
||||
|
||||
### Logging Pattern
|
||||
|
||||
All exceptions logged at throw site with scope context:
|
||||
|
||||
```csharp
|
||||
try
|
||||
{
|
||||
// Execute query
|
||||
}
|
||||
catch (OracleException ex) when (ex.Number == 1017) // Invalid credentials
|
||||
{
|
||||
using (_logger.BeginScope(new Dictionary<string, object>
|
||||
{
|
||||
["DataSource"] = "JDE",
|
||||
["Operation"] = "GetWorkOrdersAsync"
|
||||
}))
|
||||
{
|
||||
_logger.LogError(ex, "Failed to connect to JDE Oracle database");
|
||||
}
|
||||
throw new ConnectionException("JDE: Failed to connect to database", "JDE", ex);
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
|
||||
### DataAccessOptions Class
|
||||
|
||||
```csharp
|
||||
public class DataAccessOptions
|
||||
{
|
||||
public const string SectionName = "DataAccess";
|
||||
|
||||
public int DefaultTimeoutSeconds { get; set; } = 600;
|
||||
public int LotUsageTimeoutSeconds { get; set; } = 999999;
|
||||
public int MisDataTimeoutSeconds { get; set; } = 60000;
|
||||
public int RebuildIndexTimeoutSeconds { get; set; } = 600;
|
||||
public string ProductionSchema { get; set; } = "PRODDTA";
|
||||
public string ArchiveSchema { get; set; } = "ARCDTAPD";
|
||||
public string StageSchema { get; set; } = "JDESTAGE";
|
||||
}
|
||||
```
|
||||
|
||||
### Configuration Binding
|
||||
|
||||
```json
|
||||
{
|
||||
"DataAccess": {
|
||||
"DefaultTimeoutSeconds": 600,
|
||||
"LotUsageTimeoutSeconds": 999999,
|
||||
"MisDataTimeoutSeconds": 60000,
|
||||
"RebuildIndexTimeoutSeconds": 600,
|
||||
"ProductionSchema": "PRODDTA",
|
||||
"ArchiveSchema": "ARCDTAPD",
|
||||
"StageSchema": "JDESTAGE"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Service Registration
|
||||
|
||||
### AddDataAccess Extension Method
|
||||
|
||||
```csharp
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddDataAccess(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
// Bind options
|
||||
services.Configure<DataAccessOptions>(
|
||||
configuration.GetSection(DataAccessOptions.SectionName));
|
||||
|
||||
// Register connection factory (singleton)
|
||||
services.AddSingleton<IDbConnectionFactory, DbConnectionFactory>();
|
||||
|
||||
// Register repositories (scoped)
|
||||
services.AddScoped<ILotFinderRepository, LotFinderRepository>();
|
||||
services.AddScoped<IJdeRepository, JdeRepository>();
|
||||
services.AddScoped<ICmsRepository, CmsRepository>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## SQL Injection Prevention
|
||||
|
||||
### RebuildIndicesAsync Whitelist
|
||||
|
||||
Table names validated against explicit whitelist:
|
||||
|
||||
```csharp
|
||||
private static readonly HashSet<string> ValidTableNames = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"Branch", "DataUpdate", "FunctionCode", "Item", "JdeUser",
|
||||
"Lot", "LotLocation", "LotUsage_Curr", "LotUsage_Hist",
|
||||
"MisData", "OrgHierarchy", "ProfitCenter", "RouteMaster",
|
||||
"Search", "StatusCode", "WorkCenter",
|
||||
"WorkOrder_Curr", "WorkOrder_Hist",
|
||||
"WorkOrderComponent_Curr", "WorkOrderComponent_Hist",
|
||||
"WorkOrderRouting",
|
||||
"WorkOrderStep_Curr", "WorkOrderStep_Hist",
|
||||
"WorkOrderTime_Curr", "WorkOrderTime_Hist"
|
||||
};
|
||||
|
||||
public async Task RebuildIndicesAsync(string tableName, CancellationToken ct = default)
|
||||
{
|
||||
if (!ValidTableNames.Contains(tableName))
|
||||
{
|
||||
throw new ArgumentException($"Invalid table name: {tableName}", nameof(tableName));
|
||||
}
|
||||
|
||||
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
|
||||
var sql = $"ALTER INDEX ALL ON [{tableName}] REBUILD WITH (FILLFACTOR = 95)";
|
||||
await connection.ExecuteAsync(sql, commandTimeout: _options.Value.RebuildIndexTimeoutSeconds);
|
||||
}
|
||||
```
|
||||
|
||||
## NuGet Dependencies
|
||||
|
||||
### Required Packages
|
||||
|
||||
```xml
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.*" />
|
||||
<PackageReference Include="Oracle.ManagedDataAccess.Core" Version="23.4.*" />
|
||||
<PackageReference Include="Dapper" Version="2.1.*" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="9.0.*" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.*" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.*" />
|
||||
</ItemGroup>
|
||||
```
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
- Mock `IDbConnectionFactory` to return mock connections
|
||||
- Use in-memory test data for query result mapping
|
||||
- Verify exception handling scenarios
|
||||
- Test cancellation token propagation
|
||||
|
||||
### Integration Tests
|
||||
|
||||
- Use Docker containers for SQL Server and Oracle
|
||||
- Test actual query execution
|
||||
- Verify streaming behavior for large datasets
|
||||
- Test connection pooling under load
|
||||
Reference in New Issue
Block a user