Compare commits
3 Commits
9e2d763741
...
4972f998b7
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4972f998b7 | ||
|
|
7518b97b79 | ||
|
|
485c7b0c2e |
63
docs/plans/2026-02-27-unit-test-audit-design.md
Normal file
63
docs/plans/2026-02-27-unit-test-audit-design.md
Normal file
@@ -0,0 +1,63 @@
|
||||
# Unit Test Audit Extension Design
|
||||
|
||||
## Goal
|
||||
|
||||
Extend the PortTracker `audit` command to classify unit tests (not just features) by inspecting .NET test source code with Roslyn.
|
||||
|
||||
## Architecture
|
||||
|
||||
Parameterize the existing audit pipeline (`AuditCommand` + `SourceIndexer` + `FeatureClassifier`) to support both `features` and `unit_tests` tables. No new files — the same indexer and classifier logic applies to test methods.
|
||||
|
||||
## CLI Interface
|
||||
|
||||
```
|
||||
dotnet run -- audit --type features|tests|all [--source <path>] [--module <id>] [--execute]
|
||||
```
|
||||
|
||||
| Flag | Default (features) | Default (tests) |
|
||||
|------|-------------------|-----------------|
|
||||
| `--type` | `features` | — |
|
||||
| `--source` | `dotnet/src/ZB.MOM.NatsNet.Server` | `dotnet/tests/ZB.MOM.NatsNet.Server.Tests` |
|
||||
| `--output` | `reports/audit-results.csv` | `reports/audit-results-tests.csv` |
|
||||
|
||||
- `--type all` runs both sequentially.
|
||||
- `--source` override works for either type.
|
||||
|
||||
## Changes Required
|
||||
|
||||
### AuditCommand.cs
|
||||
|
||||
1. Add `--type` option with values `features`, `tests`, `all`.
|
||||
2. Thread an `AuditTarget` (table name + default source + default output + display label) through `RunAudit` and `ApplyUpdates`.
|
||||
3. `--type all` calls `RunAudit` twice with different targets.
|
||||
4. `ApplyUpdates` uses the target's table name in UPDATE SQL.
|
||||
|
||||
### FeatureClassifier.cs
|
||||
|
||||
No changes. Same N/A lookup and classification logic applies to unit tests.
|
||||
|
||||
### SourceIndexer.cs
|
||||
|
||||
No changes. Already generic — just pass a different directory path.
|
||||
|
||||
## Pre-audit DB Reset
|
||||
|
||||
Before running the test audit, manually reset deferred tests to `unknown`:
|
||||
|
||||
```sql
|
||||
sqlite3 porting.db "UPDATE unit_tests SET status = 'unknown' WHERE status = 'deferred';"
|
||||
```
|
||||
|
||||
## Execution Sequence
|
||||
|
||||
1. Reset deferred tests: `sqlite3 porting.db "UPDATE unit_tests SET status = 'unknown' WHERE status = 'deferred';"`
|
||||
2. Run audit: `dotnet run -- audit --type tests --db porting.db --execute`
|
||||
3. Verify results and generate report.
|
||||
|
||||
## Classification Behavior for Tests
|
||||
|
||||
Same priority as features:
|
||||
1. **N/A**: Go method matches logging/signal patterns → `n_a`
|
||||
2. **Method found**: Test class + method exists in test project → `verified` or `stub`
|
||||
3. **Class exists, method missing**: → `deferred` ("method not found")
|
||||
4. **Class not found**: → `deferred` ("class not found")
|
||||
BIN
porting.db
BIN
porting.db
Binary file not shown.
2681
reports/audit-results-tests.csv
Normal file
2681
reports/audit-results-tests.csv
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
# NATS .NET Porting Status Report
|
||||
|
||||
Generated: 2026-02-27 10:27:48 UTC
|
||||
Generated: 2026-02-27 10:36:34 UTC
|
||||
|
||||
## Modules (12 total)
|
||||
|
||||
@@ -21,8 +21,9 @@ Generated: 2026-02-27 10:27:48 UTC
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| deferred | 2680 |
|
||||
| deferred | 2662 |
|
||||
| n_a | 187 |
|
||||
| stub | 18 |
|
||||
| verified | 390 |
|
||||
|
||||
## Library Mappings (36 total)
|
||||
|
||||
37
reports/report_485c7b0.md
Normal file
37
reports/report_485c7b0.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# NATS .NET Porting Status Report
|
||||
|
||||
Generated: 2026-02-27 10:35:52 UTC
|
||||
|
||||
## Modules (12 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| verified | 12 |
|
||||
|
||||
## Features (3673 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| deferred | 2500 |
|
||||
| n_a | 18 |
|
||||
| stub | 168 |
|
||||
| verified | 987 |
|
||||
|
||||
## Unit Tests (3257 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| deferred | 2680 |
|
||||
| n_a | 187 |
|
||||
| verified | 390 |
|
||||
|
||||
## Library Mappings (36 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| mapped | 36 |
|
||||
|
||||
|
||||
## Overall Progress
|
||||
|
||||
**1594/6942 items complete (23.0%)**
|
||||
38
reports/report_7518b97.md
Normal file
38
reports/report_7518b97.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# NATS .NET Porting Status Report
|
||||
|
||||
Generated: 2026-02-27 10:36:34 UTC
|
||||
|
||||
## Modules (12 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| verified | 12 |
|
||||
|
||||
## Features (3673 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| deferred | 2500 |
|
||||
| n_a | 18 |
|
||||
| stub | 168 |
|
||||
| verified | 987 |
|
||||
|
||||
## Unit Tests (3257 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| deferred | 2662 |
|
||||
| n_a | 187 |
|
||||
| stub | 18 |
|
||||
| verified | 390 |
|
||||
|
||||
## Library Mappings (36 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| mapped | 36 |
|
||||
|
||||
|
||||
## Overall Progress
|
||||
|
||||
**1594/6942 items complete (23.0%)**
|
||||
37
reports/report_9e2d763.md
Normal file
37
reports/report_9e2d763.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# NATS .NET Porting Status Report
|
||||
|
||||
Generated: 2026-02-27 10:34:31 UTC
|
||||
|
||||
## Modules (12 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| verified | 12 |
|
||||
|
||||
## Features (3673 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| deferred | 2500 |
|
||||
| n_a | 18 |
|
||||
| stub | 168 |
|
||||
| verified | 987 |
|
||||
|
||||
## Unit Tests (3257 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| deferred | 2680 |
|
||||
| n_a | 187 |
|
||||
| verified | 390 |
|
||||
|
||||
## Library Mappings (36 total)
|
||||
|
||||
| Status | Count |
|
||||
|--------|-------|
|
||||
| mapped | 36 |
|
||||
|
||||
|
||||
## Overall Progress
|
||||
|
||||
**1594/6942 items complete (23.0%)**
|
||||
@@ -7,18 +7,28 @@ namespace NatsNet.PortTracker.Commands;
|
||||
|
||||
public static class AuditCommand
|
||||
{
|
||||
private record AuditTarget(string Table, string Label, string DefaultSource, string DefaultOutput);
|
||||
|
||||
private static readonly AuditTarget FeaturesTarget = new(
|
||||
"features", "features",
|
||||
Path.Combine(Directory.GetCurrentDirectory(), "dotnet", "src", "ZB.MOM.NatsNet.Server"),
|
||||
Path.Combine(Directory.GetCurrentDirectory(), "reports", "audit-results.csv"));
|
||||
|
||||
private static readonly AuditTarget TestsTarget = new(
|
||||
"unit_tests", "unit tests",
|
||||
Path.Combine(Directory.GetCurrentDirectory(), "dotnet", "tests", "ZB.MOM.NatsNet.Server.Tests"),
|
||||
Path.Combine(Directory.GetCurrentDirectory(), "reports", "audit-results-tests.csv"));
|
||||
|
||||
public static Command Create(Option<string> dbOption)
|
||||
{
|
||||
var sourceOpt = new Option<string>("--source")
|
||||
var sourceOpt = new Option<string?>("--source")
|
||||
{
|
||||
Description = "Path to the .NET source directory",
|
||||
DefaultValueFactory = _ => Path.Combine(Directory.GetCurrentDirectory(), "dotnet", "src", "ZB.MOM.NatsNet.Server")
|
||||
Description = "Path to the .NET source directory (defaults based on --type)"
|
||||
};
|
||||
|
||||
var outputOpt = new Option<string>("--output")
|
||||
var outputOpt = new Option<string?>("--output")
|
||||
{
|
||||
Description = "CSV report output path",
|
||||
DefaultValueFactory = _ => Path.Combine(Directory.GetCurrentDirectory(), "reports", "audit-results.csv")
|
||||
Description = "CSV report output path (defaults based on --type)"
|
||||
};
|
||||
|
||||
var moduleOpt = new Option<int?>("--module")
|
||||
@@ -32,44 +42,62 @@ public static class AuditCommand
|
||||
DefaultValueFactory = _ => false
|
||||
};
|
||||
|
||||
var cmd = new Command("audit", "Classify unknown features by inspecting .NET source code");
|
||||
var typeOpt = new Option<string>("--type")
|
||||
{
|
||||
Description = "What to audit: features, tests, or all",
|
||||
DefaultValueFactory = _ => "features"
|
||||
};
|
||||
|
||||
var cmd = new Command("audit", "Classify unknown features/tests by inspecting .NET source code");
|
||||
cmd.Add(sourceOpt);
|
||||
cmd.Add(outputOpt);
|
||||
cmd.Add(moduleOpt);
|
||||
cmd.Add(executeOpt);
|
||||
cmd.Add(typeOpt);
|
||||
|
||||
cmd.SetAction(parseResult =>
|
||||
{
|
||||
var dbPath = parseResult.GetValue(dbOption)!;
|
||||
var sourcePath = parseResult.GetValue(sourceOpt)!;
|
||||
var outputPath = parseResult.GetValue(outputOpt)!;
|
||||
var sourceOverride = parseResult.GetValue(sourceOpt);
|
||||
var outputOverride = parseResult.GetValue(outputOpt);
|
||||
var moduleId = parseResult.GetValue(moduleOpt);
|
||||
var execute = parseResult.GetValue(executeOpt);
|
||||
var type = parseResult.GetValue(typeOpt)!;
|
||||
|
||||
RunAudit(dbPath, sourcePath, outputPath, moduleId, execute);
|
||||
AuditTarget[] targets = type switch
|
||||
{
|
||||
"features" => [FeaturesTarget],
|
||||
"tests" => [TestsTarget],
|
||||
"all" => [FeaturesTarget, TestsTarget],
|
||||
_ => throw new ArgumentException($"Unknown audit type: {type}. Use features, tests, or all.")
|
||||
};
|
||||
|
||||
foreach (var target in targets)
|
||||
{
|
||||
var sourcePath = sourceOverride ?? target.DefaultSource;
|
||||
var outputPath = outputOverride ?? target.DefaultOutput;
|
||||
RunAudit(dbPath, sourcePath, outputPath, moduleId, execute, target);
|
||||
}
|
||||
});
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
private static void RunAudit(string dbPath, string sourcePath, string outputPath, int? moduleId, bool execute)
|
||||
private static void RunAudit(string dbPath, string sourcePath, string outputPath, int? moduleId, bool execute, AuditTarget target)
|
||||
{
|
||||
// Validate source directory
|
||||
if (!Directory.Exists(sourcePath))
|
||||
{
|
||||
Console.WriteLine($"Error: source directory not found: {sourcePath}");
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Build source index
|
||||
Console.WriteLine($"Parsing .NET source files in {sourcePath}...");
|
||||
var indexer = new SourceIndexer();
|
||||
indexer.IndexDirectory(sourcePath);
|
||||
Console.WriteLine($"Indexed {indexer.FilesIndexed} files, {indexer.MethodsIndexed} methods/properties.");
|
||||
|
||||
// 2. Query unknown features
|
||||
using var db = new Database(dbPath);
|
||||
var sql = "SELECT id, dotnet_class, dotnet_method, go_file, go_method FROM features WHERE status = 'unknown'";
|
||||
var sql = $"SELECT id, dotnet_class, dotnet_method, go_file, go_method FROM {target.Table} WHERE status = 'unknown'";
|
||||
var parameters = new List<(string, object?)>();
|
||||
if (moduleId is not null)
|
||||
{
|
||||
@@ -81,12 +109,11 @@ public static class AuditCommand
|
||||
var rows = db.Query(sql, parameters.ToArray());
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
Console.WriteLine("No unknown features found.");
|
||||
Console.WriteLine($"No unknown {target.Label} found.");
|
||||
return;
|
||||
}
|
||||
Console.WriteLine($"Found {rows.Count} unknown features to classify.\n");
|
||||
Console.WriteLine($"Found {rows.Count} unknown {target.Label} to classify.\n");
|
||||
|
||||
// 3. Classify each feature
|
||||
var classifier = new FeatureClassifier(indexer);
|
||||
var results = new List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)>();
|
||||
|
||||
@@ -103,17 +130,16 @@ public static class AuditCommand
|
||||
results.Add((feature, result));
|
||||
}
|
||||
|
||||
// 4. Write CSV report
|
||||
WriteCsvReport(outputPath, results);
|
||||
|
||||
// 5. Print console summary
|
||||
var grouped = results.GroupBy(r => r.Result.Status)
|
||||
.ToDictionary(g => g.Key, g => g.Count());
|
||||
|
||||
Console.WriteLine("Feature Status Audit Results");
|
||||
Console.WriteLine("=============================");
|
||||
var label = char.ToUpper(target.Label[0]) + target.Label[1..];
|
||||
Console.WriteLine($"{label} Status Audit Results");
|
||||
Console.WriteLine(new string('=', $"{label} Status Audit Results".Length));
|
||||
Console.WriteLine($"Source: {sourcePath} ({indexer.FilesIndexed} files, {indexer.MethodsIndexed} methods indexed)");
|
||||
Console.WriteLine($"Features audited: {results.Count}");
|
||||
Console.WriteLine($"{label} audited: {results.Count}");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($" verified: {grouped.GetValueOrDefault("verified", 0)}");
|
||||
Console.WriteLine($" stub: {grouped.GetValueOrDefault("stub", 0)}");
|
||||
@@ -128,8 +154,7 @@ public static class AuditCommand
|
||||
return;
|
||||
}
|
||||
|
||||
// 6. Apply DB updates
|
||||
ApplyUpdates(db, results);
|
||||
ApplyUpdates(db, results, target);
|
||||
Console.WriteLine($"Report: {outputPath}");
|
||||
}
|
||||
|
||||
@@ -137,7 +162,6 @@ public static class AuditCommand
|
||||
string outputPath,
|
||||
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results)
|
||||
{
|
||||
// Ensure directory exists
|
||||
var dir = Path.GetDirectoryName(outputPath);
|
||||
if (!string.IsNullOrEmpty(dir))
|
||||
Directory.CreateDirectory(dir);
|
||||
@@ -153,9 +177,9 @@ public static class AuditCommand
|
||||
|
||||
private static void ApplyUpdates(
|
||||
Database db,
|
||||
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results)
|
||||
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results,
|
||||
AuditTarget target)
|
||||
{
|
||||
// Group by (status, notes) for efficient batch updates
|
||||
var groups = results
|
||||
.GroupBy(r => (r.Result.Status, Notes: r.Result.Status == "n_a" ? r.Result.Reason : (string?)null))
|
||||
.ToList();
|
||||
@@ -170,7 +194,6 @@ public static class AuditCommand
|
||||
var status = group.Key.Status;
|
||||
var notes = group.Key.Notes;
|
||||
|
||||
// Build parameterized IN clause
|
||||
var placeholders = new List<string>();
|
||||
using var cmd = db.CreateCommand("");
|
||||
for (var i = 0; i < ids.Count; i++)
|
||||
@@ -183,12 +206,12 @@ public static class AuditCommand
|
||||
|
||||
if (notes is not null)
|
||||
{
|
||||
cmd.CommandText = $"UPDATE features SET status = @status, notes = @notes WHERE id IN ({string.Join(", ", placeholders)})";
|
||||
cmd.CommandText = $"UPDATE {target.Table} SET status = @status, notes = @notes WHERE id IN ({string.Join(", ", placeholders)})";
|
||||
cmd.Parameters.AddWithValue("@notes", notes);
|
||||
}
|
||||
else
|
||||
{
|
||||
cmd.CommandText = $"UPDATE features SET status = @status WHERE id IN ({string.Join(", ", placeholders)})";
|
||||
cmd.CommandText = $"UPDATE {target.Table} SET status = @status WHERE id IN ({string.Join(", ", placeholders)})";
|
||||
}
|
||||
|
||||
cmd.Transaction = transaction;
|
||||
@@ -197,7 +220,7 @@ public static class AuditCommand
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
Console.WriteLine($"Updated {totalUpdated} features.");
|
||||
Console.WriteLine($"Updated {totalUpdated} {target.Label}.");
|
||||
}
|
||||
catch
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user