3 Commits

Author SHA1 Message Date
Joseph Doherty
4972f998b7 feat: run unit test audit — classify 2680 deferred tests
Reset 2680 deferred unit tests to unknown and ran audit against
dotnet/tests/. Results: 18 reclassified as stub (JetStreamErrors
and SignalHandler test stubs), 2662 remain deferred.
2026-02-27 05:36:34 -05:00
Joseph Doherty
7518b97b79 feat: extend audit command to support unit tests with --type flag 2026-02-27 05:35:51 -05:00
Joseph Doherty
485c7b0c2e docs: add unit test audit extension design 2026-02-27 05:34:31 -05:00
8 changed files with 2914 additions and 34 deletions

View File

@@ -0,0 +1,63 @@
# Unit Test Audit Extension Design
## Goal
Extend the PortTracker `audit` command to classify unit tests (not just features) by inspecting .NET test source code with Roslyn.
## Architecture
Parameterize the existing audit pipeline (`AuditCommand` + `SourceIndexer` + `FeatureClassifier`) to support both `features` and `unit_tests` tables. No new files — the same indexer and classifier logic applies to test methods.
## CLI Interface
```
dotnet run -- audit --type features|tests|all [--source <path>] [--module <id>] [--execute]
```
| Flag | Default (features) | Default (tests) |
|------|-------------------|-----------------|
| `--type` | `features` | — |
| `--source` | `dotnet/src/ZB.MOM.NatsNet.Server` | `dotnet/tests/ZB.MOM.NatsNet.Server.Tests` |
| `--output` | `reports/audit-results.csv` | `reports/audit-results-tests.csv` |
- `--type all` runs both sequentially.
- `--source` override works for either type.
## Changes Required
### AuditCommand.cs
1. Add `--type` option with values `features`, `tests`, `all`.
2. Thread an `AuditTarget` (table name + default source + default output + display label) through `RunAudit` and `ApplyUpdates`.
3. `--type all` calls `RunAudit` twice with different targets.
4. `ApplyUpdates` uses the target's table name in UPDATE SQL.
### FeatureClassifier.cs
No changes. Same N/A lookup and classification logic applies to unit tests.
### SourceIndexer.cs
No changes. Already generic — just pass a different directory path.
## Pre-audit DB Reset
Before running the test audit, manually reset deferred tests to `unknown`:
```sql
sqlite3 porting.db "UPDATE unit_tests SET status = 'unknown' WHERE status = 'deferred';"
```
## Execution Sequence
1. Reset deferred tests: `sqlite3 porting.db "UPDATE unit_tests SET status = 'unknown' WHERE status = 'deferred';"`
2. Run audit: `dotnet run -- audit --type tests --db porting.db --execute`
3. Verify results and generate report.
## Classification Behavior for Tests
Same priority as features:
1. **N/A**: Go method matches logging/signal patterns → `n_a`
2. **Method found**: Test class + method exists in test project → `verified` or `stub`
3. **Class exists, method missing**: → `deferred` ("method not found")
4. **Class not found**: → `deferred` ("class not found")

Binary file not shown.

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
# NATS .NET Porting Status Report
Generated: 2026-02-27 10:27:48 UTC
Generated: 2026-02-27 10:36:34 UTC
## Modules (12 total)
@@ -21,8 +21,9 @@ Generated: 2026-02-27 10:27:48 UTC
| Status | Count |
|--------|-------|
| deferred | 2680 |
| deferred | 2662 |
| n_a | 187 |
| stub | 18 |
| verified | 390 |
## Library Mappings (36 total)

37
reports/report_485c7b0.md Normal file
View File

@@ -0,0 +1,37 @@
# NATS .NET Porting Status Report
Generated: 2026-02-27 10:35:52 UTC
## Modules (12 total)
| Status | Count |
|--------|-------|
| verified | 12 |
## Features (3673 total)
| Status | Count |
|--------|-------|
| deferred | 2500 |
| n_a | 18 |
| stub | 168 |
| verified | 987 |
## Unit Tests (3257 total)
| Status | Count |
|--------|-------|
| deferred | 2680 |
| n_a | 187 |
| verified | 390 |
## Library Mappings (36 total)
| Status | Count |
|--------|-------|
| mapped | 36 |
## Overall Progress
**1594/6942 items complete (23.0%)**

38
reports/report_7518b97.md Normal file
View File

@@ -0,0 +1,38 @@
# NATS .NET Porting Status Report
Generated: 2026-02-27 10:36:34 UTC
## Modules (12 total)
| Status | Count |
|--------|-------|
| verified | 12 |
## Features (3673 total)
| Status | Count |
|--------|-------|
| deferred | 2500 |
| n_a | 18 |
| stub | 168 |
| verified | 987 |
## Unit Tests (3257 total)
| Status | Count |
|--------|-------|
| deferred | 2662 |
| n_a | 187 |
| stub | 18 |
| verified | 390 |
## Library Mappings (36 total)
| Status | Count |
|--------|-------|
| mapped | 36 |
## Overall Progress
**1594/6942 items complete (23.0%)**

37
reports/report_9e2d763.md Normal file
View File

@@ -0,0 +1,37 @@
# NATS .NET Porting Status Report
Generated: 2026-02-27 10:34:31 UTC
## Modules (12 total)
| Status | Count |
|--------|-------|
| verified | 12 |
## Features (3673 total)
| Status | Count |
|--------|-------|
| deferred | 2500 |
| n_a | 18 |
| stub | 168 |
| verified | 987 |
## Unit Tests (3257 total)
| Status | Count |
|--------|-------|
| deferred | 2680 |
| n_a | 187 |
| verified | 390 |
## Library Mappings (36 total)
| Status | Count |
|--------|-------|
| mapped | 36 |
## Overall Progress
**1594/6942 items complete (23.0%)**

View File

@@ -7,18 +7,28 @@ namespace NatsNet.PortTracker.Commands;
public static class AuditCommand
{
private record AuditTarget(string Table, string Label, string DefaultSource, string DefaultOutput);
private static readonly AuditTarget FeaturesTarget = new(
"features", "features",
Path.Combine(Directory.GetCurrentDirectory(), "dotnet", "src", "ZB.MOM.NatsNet.Server"),
Path.Combine(Directory.GetCurrentDirectory(), "reports", "audit-results.csv"));
private static readonly AuditTarget TestsTarget = new(
"unit_tests", "unit tests",
Path.Combine(Directory.GetCurrentDirectory(), "dotnet", "tests", "ZB.MOM.NatsNet.Server.Tests"),
Path.Combine(Directory.GetCurrentDirectory(), "reports", "audit-results-tests.csv"));
public static Command Create(Option<string> dbOption)
{
var sourceOpt = new Option<string>("--source")
var sourceOpt = new Option<string?>("--source")
{
Description = "Path to the .NET source directory",
DefaultValueFactory = _ => Path.Combine(Directory.GetCurrentDirectory(), "dotnet", "src", "ZB.MOM.NatsNet.Server")
Description = "Path to the .NET source directory (defaults based on --type)"
};
var outputOpt = new Option<string>("--output")
var outputOpt = new Option<string?>("--output")
{
Description = "CSV report output path",
DefaultValueFactory = _ => Path.Combine(Directory.GetCurrentDirectory(), "reports", "audit-results.csv")
Description = "CSV report output path (defaults based on --type)"
};
var moduleOpt = new Option<int?>("--module")
@@ -32,44 +42,62 @@ public static class AuditCommand
DefaultValueFactory = _ => false
};
var cmd = new Command("audit", "Classify unknown features by inspecting .NET source code");
var typeOpt = new Option<string>("--type")
{
Description = "What to audit: features, tests, or all",
DefaultValueFactory = _ => "features"
};
var cmd = new Command("audit", "Classify unknown features/tests by inspecting .NET source code");
cmd.Add(sourceOpt);
cmd.Add(outputOpt);
cmd.Add(moduleOpt);
cmd.Add(executeOpt);
cmd.Add(typeOpt);
cmd.SetAction(parseResult =>
{
var dbPath = parseResult.GetValue(dbOption)!;
var sourcePath = parseResult.GetValue(sourceOpt)!;
var outputPath = parseResult.GetValue(outputOpt)!;
var sourceOverride = parseResult.GetValue(sourceOpt);
var outputOverride = parseResult.GetValue(outputOpt);
var moduleId = parseResult.GetValue(moduleOpt);
var execute = parseResult.GetValue(executeOpt);
var type = parseResult.GetValue(typeOpt)!;
RunAudit(dbPath, sourcePath, outputPath, moduleId, execute);
AuditTarget[] targets = type switch
{
"features" => [FeaturesTarget],
"tests" => [TestsTarget],
"all" => [FeaturesTarget, TestsTarget],
_ => throw new ArgumentException($"Unknown audit type: {type}. Use features, tests, or all.")
};
foreach (var target in targets)
{
var sourcePath = sourceOverride ?? target.DefaultSource;
var outputPath = outputOverride ?? target.DefaultOutput;
RunAudit(dbPath, sourcePath, outputPath, moduleId, execute, target);
}
});
return cmd;
}
private static void RunAudit(string dbPath, string sourcePath, string outputPath, int? moduleId, bool execute)
private static void RunAudit(string dbPath, string sourcePath, string outputPath, int? moduleId, bool execute, AuditTarget target)
{
// Validate source directory
if (!Directory.Exists(sourcePath))
{
Console.WriteLine($"Error: source directory not found: {sourcePath}");
return;
}
// 1. Build source index
Console.WriteLine($"Parsing .NET source files in {sourcePath}...");
var indexer = new SourceIndexer();
indexer.IndexDirectory(sourcePath);
Console.WriteLine($"Indexed {indexer.FilesIndexed} files, {indexer.MethodsIndexed} methods/properties.");
// 2. Query unknown features
using var db = new Database(dbPath);
var sql = "SELECT id, dotnet_class, dotnet_method, go_file, go_method FROM features WHERE status = 'unknown'";
var sql = $"SELECT id, dotnet_class, dotnet_method, go_file, go_method FROM {target.Table} WHERE status = 'unknown'";
var parameters = new List<(string, object?)>();
if (moduleId is not null)
{
@@ -81,12 +109,11 @@ public static class AuditCommand
var rows = db.Query(sql, parameters.ToArray());
if (rows.Count == 0)
{
Console.WriteLine("No unknown features found.");
Console.WriteLine($"No unknown {target.Label} found.");
return;
}
Console.WriteLine($"Found {rows.Count} unknown features to classify.\n");
Console.WriteLine($"Found {rows.Count} unknown {target.Label} to classify.\n");
// 3. Classify each feature
var classifier = new FeatureClassifier(indexer);
var results = new List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)>();
@@ -103,17 +130,16 @@ public static class AuditCommand
results.Add((feature, result));
}
// 4. Write CSV report
WriteCsvReport(outputPath, results);
// 5. Print console summary
var grouped = results.GroupBy(r => r.Result.Status)
.ToDictionary(g => g.Key, g => g.Count());
Console.WriteLine("Feature Status Audit Results");
Console.WriteLine("=============================");
var label = char.ToUpper(target.Label[0]) + target.Label[1..];
Console.WriteLine($"{label} Status Audit Results");
Console.WriteLine(new string('=', $"{label} Status Audit Results".Length));
Console.WriteLine($"Source: {sourcePath} ({indexer.FilesIndexed} files, {indexer.MethodsIndexed} methods indexed)");
Console.WriteLine($"Features audited: {results.Count}");
Console.WriteLine($"{label} audited: {results.Count}");
Console.WriteLine();
Console.WriteLine($" verified: {grouped.GetValueOrDefault("verified", 0)}");
Console.WriteLine($" stub: {grouped.GetValueOrDefault("stub", 0)}");
@@ -128,8 +154,7 @@ public static class AuditCommand
return;
}
// 6. Apply DB updates
ApplyUpdates(db, results);
ApplyUpdates(db, results, target);
Console.WriteLine($"Report: {outputPath}");
}
@@ -137,7 +162,6 @@ public static class AuditCommand
string outputPath,
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results)
{
// Ensure directory exists
var dir = Path.GetDirectoryName(outputPath);
if (!string.IsNullOrEmpty(dir))
Directory.CreateDirectory(dir);
@@ -153,9 +177,9 @@ public static class AuditCommand
private static void ApplyUpdates(
Database db,
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results)
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results,
AuditTarget target)
{
// Group by (status, notes) for efficient batch updates
var groups = results
.GroupBy(r => (r.Result.Status, Notes: r.Result.Status == "n_a" ? r.Result.Reason : (string?)null))
.ToList();
@@ -170,7 +194,6 @@ public static class AuditCommand
var status = group.Key.Status;
var notes = group.Key.Notes;
// Build parameterized IN clause
var placeholders = new List<string>();
using var cmd = db.CreateCommand("");
for (var i = 0; i < ids.Count; i++)
@@ -183,12 +206,12 @@ public static class AuditCommand
if (notes is not null)
{
cmd.CommandText = $"UPDATE features SET status = @status, notes = @notes WHERE id IN ({string.Join(", ", placeholders)})";
cmd.CommandText = $"UPDATE {target.Table} SET status = @status, notes = @notes WHERE id IN ({string.Join(", ", placeholders)})";
cmd.Parameters.AddWithValue("@notes", notes);
}
else
{
cmd.CommandText = $"UPDATE features SET status = @status WHERE id IN ({string.Join(", ", placeholders)})";
cmd.CommandText = $"UPDATE {target.Table} SET status = @status WHERE id IN ({string.Join(", ", placeholders)})";
}
cmd.Transaction = transaction;
@@ -197,7 +220,7 @@ public static class AuditCommand
}
transaction.Commit();
Console.WriteLine($"Updated {totalUpdated} features.");
Console.WriteLine($"Updated {totalUpdated} {target.Label}.");
}
catch
{