Joseph Doherty
26ff8d9b4f
Set up repository with legacy .NET Framework 4.8 source (OLD/), new .NET 10 Blazor solution (NEW/), OpenSpec specifications, documentation, and project configuration.
519 lines
18 KiB
Markdown
519 lines
18 KiB
Markdown
# Search Processing Design
|
|
|
|
## Overview
|
|
|
|
This document describes the architecture and implementation approach for the search processing subsystem, including the SqlKata query builder pattern, filter handler architecture, and search processor service.
|
|
|
|
## Architecture
|
|
|
|
### High-Level Component Diagram
|
|
|
|
```
|
|
┌─────────────────────────────────────────────────────────────────────────┐
|
|
│ SearchProcessor │
|
|
│ - Orchestrates search execution │
|
|
│ - Coordinates filter enrichment, query building, execution │
|
|
└────────────────────────────────┬────────────────────────────────────────┘
|
|
│
|
|
┌───────────────────────┼───────────────────────┐
|
|
│ │ │
|
|
▼ ▼ ▼
|
|
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
|
│ ILotFinder │ │ ISearchQuery │ │ IWorkOrder │
|
|
│ Repository │ │ Builder │ │ TraversalService│
|
|
│ (enrichment) │ │ (SQL generation)│ │ (downstream) │
|
|
└─────────────────┘ └────────┬────────┘ └─────────────────┘
|
|
│
|
|
┌───────────┴───────────┐
|
|
│ │
|
|
▼ ▼
|
|
┌─────────────────┐ ┌─────────────────┐
|
|
│ Filter Handlers │ │ SqlServerCompiler│
|
|
│ (composable) │ │ (SQL output) │
|
|
└─────────────────┘ └─────────────────┘
|
|
```
|
|
|
|
### Project Structure
|
|
|
|
```
|
|
NEW/src/JdeScoping.SearchProcessing/
|
|
├── Interfaces/
|
|
│ ├── ISearchProcessor.cs
|
|
│ ├── ISearchQueryBuilder.cs
|
|
│ ├── IFilterHandler.cs
|
|
│ └── IWorkOrderTraversalService.cs
|
|
├── QueryBuilders/
|
|
│ ├── SqlKataSearchQueryBuilder.cs
|
|
│ └── MisQueryBuilder.cs
|
|
├── FilterHandlers/
|
|
│ ├── FilterHandlerBase.cs
|
|
│ ├── WorkOrderFilterHandler.cs
|
|
│ ├── ItemNumberFilterHandler.cs
|
|
│ ├── ProfitCenterFilterHandler.cs
|
|
│ ├── WorkCenterFilterHandler.cs
|
|
│ ├── OperatorFilterHandler.cs
|
|
│ ├── ComponentLotFilterHandler.cs
|
|
│ ├── ItemOperationMisFilterHandler.cs
|
|
│ └── TimespanFilterHandler.cs
|
|
├── Models/
|
|
│ ├── SearchModel.cs
|
|
│ ├── SearchQueryResult.cs
|
|
│ ├── FilterEntries/
|
|
│ │ ├── WorkOrderFilterEntry.cs
|
|
│ │ ├── ItemNumberFilterEntry.cs
|
|
│ │ ├── ProfitCenterFilterEntry.cs
|
|
│ │ ├── WorkCenterFilterEntry.cs
|
|
│ │ ├── OperatorFilterEntry.cs
|
|
│ │ ├── ComponentLotFilterEntry.cs
|
|
│ │ └── ItemOperationMisFilterEntry.cs
|
|
│ └── Results/
|
|
│ ├── SearchResult.cs
|
|
│ ├── MisSearchResult.cs
|
|
│ └── MisNonMatchSearchResult.cs
|
|
├── Services/
|
|
│ ├── SearchProcessor.cs
|
|
│ └── WorkOrderTraversalService.cs
|
|
├── Configuration/
|
|
│ └── SearchProcessingOptions.cs
|
|
├── Attributes/
|
|
│ ├── OutputColumnAttribute.cs
|
|
│ └── OutputTableAttribute.cs
|
|
├── Extensions/
|
|
│ ├── SearchModelExtensions.cs
|
|
│ └── TableValuedParameterExtensions.cs
|
|
├── ServiceCollectionExtensions.cs
|
|
└── JdeScoping.SearchProcessing.csproj
|
|
```
|
|
|
|
## SqlKata Query Builder Architecture
|
|
|
|
### ISearchQueryBuilder Interface
|
|
|
|
```csharp
|
|
public interface ISearchQueryBuilder
|
|
{
|
|
/// <summary>
|
|
/// Builds the main search query for flagging and retrieving work orders.
|
|
/// </summary>
|
|
SearchQueryResult BuildSearchQuery(SearchModel model);
|
|
|
|
/// <summary>
|
|
/// Builds the MIS data extraction query when ExtractMisData is enabled.
|
|
/// </summary>
|
|
SearchQueryResult BuildMisQuery(SearchModel model);
|
|
|
|
/// <summary>
|
|
/// Builds the MIS non-match query for work orders without MIS records.
|
|
/// </summary>
|
|
SearchQueryResult BuildMisNonMatchQuery(SearchModel model);
|
|
}
|
|
|
|
public record SearchQueryResult(
|
|
string Sql,
|
|
IDictionary<string, object> Parameters,
|
|
IReadOnlyList<string> TempTableSetupSql);
|
|
```
|
|
|
|
### SqlKata Integration Pattern
|
|
|
|
The SqlKata query builder composes queries using fluent methods:
|
|
|
|
```csharp
|
|
public sealed class SqlKataSearchQueryBuilder : ISearchQueryBuilder
|
|
{
|
|
private readonly SqlServerCompiler _compiler = new();
|
|
private readonly IEnumerable<IFilterHandler> _filterHandlers;
|
|
|
|
public SqlKataSearchQueryBuilder(IEnumerable<IFilterHandler> filterHandlers)
|
|
{
|
|
_filterHandlers = filterHandlers;
|
|
}
|
|
|
|
public SearchQueryResult BuildSearchQuery(SearchModel model)
|
|
{
|
|
var setupStatements = new List<string>();
|
|
var parameters = new Dictionary<string, object>();
|
|
|
|
// Build temp table setup SQL
|
|
setupStatements.Add(BuildTempWoTableSql());
|
|
|
|
// Apply filter handlers (each may add setup SQL and parameters)
|
|
foreach (var handler in _filterHandlers.Where(h => h.IsEnabled(model)))
|
|
{
|
|
var filterResult = handler.Apply(model, _compiler);
|
|
setupStatements.AddRange(filterResult.SetupSql);
|
|
foreach (var param in filterResult.Parameters)
|
|
{
|
|
parameters[param.Key] = param.Value;
|
|
}
|
|
}
|
|
|
|
// Build final result query
|
|
var resultQuery = BuildResultQuery();
|
|
var compiled = _compiler.Compile(resultQuery);
|
|
|
|
return new SearchQueryResult(
|
|
compiled.Sql,
|
|
MergeParameters(parameters, compiled.NamedBindings),
|
|
setupStatements);
|
|
}
|
|
}
|
|
```
|
|
|
|
### Why SqlKata Instead of T4 Templates
|
|
|
|
| Aspect | T4 Template (Legacy) | SqlKata (New) |
|
|
|--------|---------------------|---------------|
|
|
| Testability | Cannot unit test | Test query structure without DB |
|
|
| Type Safety | String concatenation | Fluent API with IntelliSense |
|
|
| SQL Injection | Manual parameter handling | Parameterized by default |
|
|
| Maintenance | Edit .tt file, regenerate | Edit C# code directly |
|
|
| SDK Support | Limited in modern .NET | Full .NET 10 support |
|
|
| Composability | Monolithic template | Pluggable filter handlers |
|
|
|
|
## Filter Handler Pattern
|
|
|
|
### IFilterHandler Interface
|
|
|
|
```csharp
|
|
public interface IFilterHandler
|
|
{
|
|
/// <summary>
|
|
/// Determines if this filter is active for the given search model.
|
|
/// </summary>
|
|
bool IsEnabled(SearchModel model);
|
|
|
|
/// <summary>
|
|
/// Applies the filter, returning setup SQL and parameters.
|
|
/// </summary>
|
|
FilterResult Apply(SearchModel model, SqlServerCompiler compiler);
|
|
|
|
/// <summary>
|
|
/// Priority for handler execution order (lower = earlier).
|
|
/// </summary>
|
|
int Priority { get; }
|
|
}
|
|
|
|
public record FilterResult(
|
|
IReadOnlyList<string> SetupSql,
|
|
IDictionary<string, object> Parameters);
|
|
```
|
|
|
|
### Filter Handler Implementations
|
|
|
|
Each filter handler encapsulates the logic for one filter type:
|
|
|
|
#### WorkOrderFilterHandler
|
|
|
|
```csharp
|
|
public sealed class WorkOrderFilterHandler : FilterHandlerBase
|
|
{
|
|
public override int Priority => 10;
|
|
|
|
public override bool IsEnabled(SearchModel model)
|
|
=> model.WorkOrderFilterEnabled;
|
|
|
|
public override FilterResult Apply(SearchModel model, SqlServerCompiler compiler)
|
|
{
|
|
// Generates MERGE into #Temp_WO with ManuallySpecified = 1
|
|
// Followed by split order detection
|
|
var sql = BuildWorkOrderMergeSql();
|
|
var parameters = new Dictionary<string, object>
|
|
{
|
|
["p_WorkOrderFilter"] = model.CreateWorkOrderFilterParameter()
|
|
};
|
|
|
|
return new FilterResult(new[] { sql }, parameters);
|
|
}
|
|
}
|
|
```
|
|
|
|
#### ComponentLotFilterHandler
|
|
|
|
```csharp
|
|
public sealed class ComponentLotFilterHandler : FilterHandlerBase
|
|
{
|
|
public override int Priority => 30;
|
|
|
|
public override bool IsEnabled(SearchModel model)
|
|
=> model.ComponentLotFilterEnabled;
|
|
|
|
public override FilterResult Apply(SearchModel model, SqlServerCompiler compiler)
|
|
{
|
|
// Joins Lot -> WorkOrderComponent/LotUsage -> WorkOrder
|
|
// Sets CARDEX = 1 flag
|
|
var sql = BuildComponentLotMergeSql();
|
|
var parameters = new Dictionary<string, object>
|
|
{
|
|
["p_ComponentLotFilter"] = model.CreateComponentLotFilterParameter()
|
|
};
|
|
|
|
return new FilterResult(new[] { sql }, parameters);
|
|
}
|
|
}
|
|
```
|
|
|
|
### Handler Execution Order
|
|
|
|
Handlers execute in priority order to ensure dependent temp tables exist:
|
|
|
|
| Priority | Handler | Creates/Uses |
|
|
|----------|---------|--------------|
|
|
| 10 | WorkOrderFilterHandler | Creates #Temp_WO entries |
|
|
| 20 | ItemNumberFilterHandler | Creates #P_ItemNumbers |
|
|
| 30 | ComponentLotFilterHandler | Uses Lot, creates #Temp_WO entries |
|
|
| 40 | ProfitCenterFilterHandler | Creates #P_WorkCenters |
|
|
| 50 | WorkCenterFilterHandler | Creates #P_WorkCenters |
|
|
| 60 | OperatorFilterHandler | Creates #P_OperatorIDs |
|
|
| 70 | ItemOperationMisFilterHandler | Creates #P_PartOperations |
|
|
| 80 | TimespanFilterHandler | Adds WHERE clause |
|
|
|
|
## IAsyncEnumerable Streaming Pattern
|
|
|
|
### Large Result Set Handling
|
|
|
|
For searches returning thousands of work orders, streaming avoids loading all results into memory:
|
|
|
|
```csharp
|
|
public interface ISearchProcessor
|
|
{
|
|
/// <summary>
|
|
/// Executes search and returns results as async stream.
|
|
/// </summary>
|
|
IAsyncEnumerable<SearchResult> ExecuteSearchAsync(
|
|
SearchModel model,
|
|
CancellationToken ct = default);
|
|
|
|
/// <summary>
|
|
/// Executes search and materializes all results into SearchModel.
|
|
/// </summary>
|
|
Task<SearchModel> ExecuteSearchToModelAsync(
|
|
SearchModel model,
|
|
CancellationToken ct = default);
|
|
}
|
|
```
|
|
|
|
### Streaming Implementation
|
|
|
|
```csharp
|
|
public sealed class SearchProcessor : ISearchProcessor
|
|
{
|
|
private readonly IDbConnectionFactory _connectionFactory;
|
|
private readonly ISearchQueryBuilder _queryBuilder;
|
|
private readonly IWorkOrderTraversalService _traversalService;
|
|
private readonly ILogger<SearchProcessor> _logger;
|
|
|
|
public async IAsyncEnumerable<SearchResult> ExecuteSearchAsync(
|
|
SearchModel model,
|
|
[EnumeratorCancellation] CancellationToken ct = default)
|
|
{
|
|
await using var connection = await _connectionFactory
|
|
.CreateLotFinderConnectionAsync(ct);
|
|
|
|
// Execute setup SQL (temp tables, filter population)
|
|
var queryResult = _queryBuilder.BuildSearchQuery(model);
|
|
foreach (var setupSql in queryResult.TempTableSetupSql)
|
|
{
|
|
await connection.ExecuteAsync(setupSql, queryResult.Parameters);
|
|
}
|
|
|
|
// Stream results
|
|
await foreach (var result in connection.QueryUnbufferedAsync<SearchResult>(
|
|
queryResult.Sql,
|
|
queryResult.Parameters)
|
|
.WithCancellation(ct))
|
|
{
|
|
yield return result;
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
## Downstream Work Order Traversal
|
|
|
|
### Stored Procedure Approach
|
|
|
|
The iterative traversal logic (up to 20 iterations finding downstream work orders) is better suited to a stored procedure:
|
|
|
|
```csharp
|
|
public interface IWorkOrderTraversalService
|
|
{
|
|
/// <summary>
|
|
/// Traverses downstream work orders via stored procedure.
|
|
/// Called after initial filtering to find related work orders.
|
|
/// </summary>
|
|
Task TraverseDownstreamAsync(
|
|
SqlConnection connection,
|
|
CancellationToken ct = default);
|
|
}
|
|
```
|
|
|
|
### Why Stored Procedure
|
|
|
|
- **Iterative logic**: WHILE loops with temp table operations are efficient in T-SQL
|
|
- **Reduced round trips**: Single call instead of 20+ iterations from C#
|
|
- **Transaction scope**: All MERGE operations in same transaction
|
|
- **Legacy compatibility**: Mirrors existing QueryTemplate.tt behavior
|
|
|
|
## Table-Valued Parameter Helpers
|
|
|
|
### Extension Methods
|
|
|
|
```csharp
|
|
public static class TableValuedParameterExtensions
|
|
{
|
|
public static SqlMapper.ICustomQueryParameter CreateWorkOrderFilterParameter(
|
|
this SearchModel model)
|
|
{
|
|
var dataTable = new DataTable();
|
|
dataTable.Columns.Add("WorkOrderNumber", typeof(long));
|
|
|
|
foreach (var entry in model.WorkOrderFilter)
|
|
{
|
|
dataTable.Rows.Add(entry.WorkOrderNumber);
|
|
}
|
|
|
|
return dataTable.AsTableValuedParameter("WorkOrderFilterParameter");
|
|
}
|
|
|
|
// Similar methods for all 7 filter types...
|
|
}
|
|
```
|
|
|
|
### TVP Type Mapping
|
|
|
|
| C# Method | SQL Server Type | Columns |
|
|
|-----------|-----------------|---------|
|
|
| `CreateWorkOrderFilterParameter` | `WorkOrderFilterParameter` | `WorkOrderNumber BIGINT` |
|
|
| `CreateItemNumberFilterParameter` | `ItemNumberFilterParameter` | `ItemNumber VARCHAR(25)` |
|
|
| `CreateProfitCenterFilterParameter` | `ProfitCenterFilterParameter` | `Code VARCHAR(12)` |
|
|
| `CreateWorkCenterFilterParameter` | `WorkCenterFilterParameter` | `Code VARCHAR(12)` |
|
|
| `CreateOperatorFilterParameter` | `OperatorFilterParameter` | `UserName VARCHAR(10)` |
|
|
| `CreateComponentLotFilterParameter` | `ComponentLotFilterParameter` | `ComponentLotNumber VARCHAR(30), ItemNumber VARCHAR(25)` |
|
|
| `CreateItemOperationMisFilterParameter` | `ItemOperationMisFilterParameter` | `ItemNumber, OperationNumber, MisNumber, MisRevision` |
|
|
|
|
## Configuration Options
|
|
|
|
### SearchProcessingOptions Class
|
|
|
|
```csharp
|
|
public class SearchProcessingOptions
|
|
{
|
|
public const string SectionName = "SearchProcessing";
|
|
|
|
/// <summary>
|
|
/// Query timeout in seconds for search execution.
|
|
/// </summary>
|
|
public int QueryTimeoutSeconds { get; set; } = 600;
|
|
|
|
/// <summary>
|
|
/// Maximum downstream traversal iterations.
|
|
/// </summary>
|
|
public int MaxTraversalIterations { get; set; } = 20;
|
|
|
|
/// <summary>
|
|
/// Enable debug SQL logging.
|
|
/// </summary>
|
|
public bool EnableDebugSql { get; set; } = false;
|
|
|
|
/// <summary>
|
|
/// Path to write debug SQL files (when EnableDebugSql is true).
|
|
/// </summary>
|
|
public string? DebugSqlPath { get; set; }
|
|
}
|
|
```
|
|
|
|
### Configuration Binding
|
|
|
|
```json
|
|
{
|
|
"SearchProcessing": {
|
|
"QueryTimeoutSeconds": 600,
|
|
"MaxTraversalIterations": 20,
|
|
"EnableDebugSql": false,
|
|
"DebugSqlPath": null
|
|
}
|
|
}
|
|
```
|
|
|
|
## Service Registration
|
|
|
|
### AddSearchProcessing Extension Method
|
|
|
|
```csharp
|
|
public static class ServiceCollectionExtensions
|
|
{
|
|
public static IServiceCollection AddSearchProcessing(
|
|
this IServiceCollection services,
|
|
IConfiguration configuration)
|
|
{
|
|
// Bind options
|
|
services.Configure<SearchProcessingOptions>(
|
|
configuration.GetSection(SearchProcessingOptions.SectionName));
|
|
|
|
// Register SqlKata compiler (singleton, thread-safe)
|
|
services.AddSingleton<SqlServerCompiler>();
|
|
|
|
// Register filter handlers (scoped)
|
|
services.AddScoped<IFilterHandler, WorkOrderFilterHandler>();
|
|
services.AddScoped<IFilterHandler, ItemNumberFilterHandler>();
|
|
services.AddScoped<IFilterHandler, ProfitCenterFilterHandler>();
|
|
services.AddScoped<IFilterHandler, WorkCenterFilterHandler>();
|
|
services.AddScoped<IFilterHandler, OperatorFilterHandler>();
|
|
services.AddScoped<IFilterHandler, ComponentLotFilterHandler>();
|
|
services.AddScoped<IFilterHandler, ItemOperationMisFilterHandler>();
|
|
services.AddScoped<IFilterHandler, TimespanFilterHandler>();
|
|
|
|
// Register query builders (scoped)
|
|
services.AddScoped<ISearchQueryBuilder, SqlKataSearchQueryBuilder>();
|
|
|
|
// Register services (scoped)
|
|
services.AddScoped<ISearchProcessor, SearchProcessor>();
|
|
services.AddScoped<IWorkOrderTraversalService, WorkOrderTraversalService>();
|
|
|
|
return services;
|
|
}
|
|
}
|
|
```
|
|
|
|
## Testing Strategy
|
|
|
|
### Unit Tests
|
|
|
|
- **Query Builder Tests**: Verify generated SQL structure without executing
|
|
- **Filter Handler Tests**: Test each handler in isolation
|
|
- **Parameter Tests**: Verify TVP creation for all filter types
|
|
- **Mock Repository**: Use NSubstitute for `ILotFinderRepository`
|
|
|
|
### Integration Tests
|
|
|
|
- **Full Search Flow**: Execute search against test database
|
|
- **Filter Combinations**: Matrix of filter permutations
|
|
- **Large Result Sets**: Verify streaming behavior
|
|
- **MIS Extraction**: Test with and without MIS data
|
|
|
|
### Test Frameworks
|
|
|
|
- **xUnit**: Test framework
|
|
- **Shouldly**: Fluent assertions
|
|
- **NSubstitute**: Mocking framework
|
|
|
|
## NuGet Dependencies
|
|
|
|
```xml
|
|
<ItemGroup>
|
|
<PackageReference Include="SqlKata" Version="3.0.*" />
|
|
<PackageReference Include="SqlKata.Execution" Version="3.0.*" />
|
|
<PackageReference Include="Dapper" Version="2.1.*" />
|
|
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.*" />
|
|
<PackageReference Include="Microsoft.Extensions.Options" Version="9.0.*" />
|
|
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.*" />
|
|
</ItemGroup>
|
|
|
|
<ItemGroup Condition="'$(Configuration)' == 'Test'">
|
|
<PackageReference Include="xunit" Version="2.9.*" />
|
|
<PackageReference Include="Shouldly" Version="4.2.*" />
|
|
<PackageReference Include="NSubstitute" Version="5.1.*" />
|
|
</ItemGroup>
|
|
```
|