Compare commits
3 Commits
9e2d763741
...
4972f998b7
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4972f998b7 | ||
|
|
7518b97b79 | ||
|
|
485c7b0c2e |
63
docs/plans/2026-02-27-unit-test-audit-design.md
Normal file
63
docs/plans/2026-02-27-unit-test-audit-design.md
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
# Unit Test Audit Extension Design
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Extend the PortTracker `audit` command to classify unit tests (not just features) by inspecting .NET test source code with Roslyn.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
Parameterize the existing audit pipeline (`AuditCommand` + `SourceIndexer` + `FeatureClassifier`) to support both `features` and `unit_tests` tables. No new files — the same indexer and classifier logic applies to test methods.
|
||||||
|
|
||||||
|
## CLI Interface
|
||||||
|
|
||||||
|
```
|
||||||
|
dotnet run -- audit --type features|tests|all [--source <path>] [--module <id>] [--execute]
|
||||||
|
```
|
||||||
|
|
||||||
|
| Flag | Default (features) | Default (tests) |
|
||||||
|
|------|-------------------|-----------------|
|
||||||
|
| `--type` | `features` | — |
|
||||||
|
| `--source` | `dotnet/src/ZB.MOM.NatsNet.Server` | `dotnet/tests/ZB.MOM.NatsNet.Server.Tests` |
|
||||||
|
| `--output` | `reports/audit-results.csv` | `reports/audit-results-tests.csv` |
|
||||||
|
|
||||||
|
- `--type all` runs both sequentially.
|
||||||
|
- `--source` override works for either type.
|
||||||
|
|
||||||
|
## Changes Required
|
||||||
|
|
||||||
|
### AuditCommand.cs
|
||||||
|
|
||||||
|
1. Add `--type` option with values `features`, `tests`, `all`.
|
||||||
|
2. Thread an `AuditTarget` (table name + default source + default output + display label) through `RunAudit` and `ApplyUpdates`.
|
||||||
|
3. `--type all` calls `RunAudit` twice with different targets.
|
||||||
|
4. `ApplyUpdates` uses the target's table name in UPDATE SQL.
|
||||||
|
|
||||||
|
### FeatureClassifier.cs
|
||||||
|
|
||||||
|
No changes. Same N/A lookup and classification logic applies to unit tests.
|
||||||
|
|
||||||
|
### SourceIndexer.cs
|
||||||
|
|
||||||
|
No changes. Already generic — just pass a different directory path.
|
||||||
|
|
||||||
|
## Pre-audit DB Reset
|
||||||
|
|
||||||
|
Before running the test audit, manually reset deferred tests to `unknown`:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
sqlite3 porting.db "UPDATE unit_tests SET status = 'unknown' WHERE status = 'deferred';"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Execution Sequence
|
||||||
|
|
||||||
|
1. Reset deferred tests: `sqlite3 porting.db "UPDATE unit_tests SET status = 'unknown' WHERE status = 'deferred';"`
|
||||||
|
2. Run audit: `dotnet run -- audit --type tests --db porting.db --execute`
|
||||||
|
3. Verify results and generate report.
|
||||||
|
|
||||||
|
## Classification Behavior for Tests
|
||||||
|
|
||||||
|
Same priority as features:
|
||||||
|
1. **N/A**: Go method matches logging/signal patterns → `n_a`
|
||||||
|
2. **Method found**: Test class + method exists in test project → `verified` or `stub`
|
||||||
|
3. **Class exists, method missing**: → `deferred` ("method not found")
|
||||||
|
4. **Class not found**: → `deferred` ("class not found")
|
||||||
BIN
porting.db
BIN
porting.db
Binary file not shown.
2681
reports/audit-results-tests.csv
Normal file
2681
reports/audit-results-tests.csv
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
|||||||
# NATS .NET Porting Status Report
|
# NATS .NET Porting Status Report
|
||||||
|
|
||||||
Generated: 2026-02-27 10:27:48 UTC
|
Generated: 2026-02-27 10:36:34 UTC
|
||||||
|
|
||||||
## Modules (12 total)
|
## Modules (12 total)
|
||||||
|
|
||||||
@@ -21,8 +21,9 @@ Generated: 2026-02-27 10:27:48 UTC
|
|||||||
|
|
||||||
| Status | Count |
|
| Status | Count |
|
||||||
|--------|-------|
|
|--------|-------|
|
||||||
| deferred | 2680 |
|
| deferred | 2662 |
|
||||||
| n_a | 187 |
|
| n_a | 187 |
|
||||||
|
| stub | 18 |
|
||||||
| verified | 390 |
|
| verified | 390 |
|
||||||
|
|
||||||
## Library Mappings (36 total)
|
## Library Mappings (36 total)
|
||||||
|
|||||||
37
reports/report_485c7b0.md
Normal file
37
reports/report_485c7b0.md
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# NATS .NET Porting Status Report
|
||||||
|
|
||||||
|
Generated: 2026-02-27 10:35:52 UTC
|
||||||
|
|
||||||
|
## Modules (12 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| verified | 12 |
|
||||||
|
|
||||||
|
## Features (3673 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| deferred | 2500 |
|
||||||
|
| n_a | 18 |
|
||||||
|
| stub | 168 |
|
||||||
|
| verified | 987 |
|
||||||
|
|
||||||
|
## Unit Tests (3257 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| deferred | 2680 |
|
||||||
|
| n_a | 187 |
|
||||||
|
| verified | 390 |
|
||||||
|
|
||||||
|
## Library Mappings (36 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| mapped | 36 |
|
||||||
|
|
||||||
|
|
||||||
|
## Overall Progress
|
||||||
|
|
||||||
|
**1594/6942 items complete (23.0%)**
|
||||||
38
reports/report_7518b97.md
Normal file
38
reports/report_7518b97.md
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
# NATS .NET Porting Status Report
|
||||||
|
|
||||||
|
Generated: 2026-02-27 10:36:34 UTC
|
||||||
|
|
||||||
|
## Modules (12 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| verified | 12 |
|
||||||
|
|
||||||
|
## Features (3673 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| deferred | 2500 |
|
||||||
|
| n_a | 18 |
|
||||||
|
| stub | 168 |
|
||||||
|
| verified | 987 |
|
||||||
|
|
||||||
|
## Unit Tests (3257 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| deferred | 2662 |
|
||||||
|
| n_a | 187 |
|
||||||
|
| stub | 18 |
|
||||||
|
| verified | 390 |
|
||||||
|
|
||||||
|
## Library Mappings (36 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| mapped | 36 |
|
||||||
|
|
||||||
|
|
||||||
|
## Overall Progress
|
||||||
|
|
||||||
|
**1594/6942 items complete (23.0%)**
|
||||||
37
reports/report_9e2d763.md
Normal file
37
reports/report_9e2d763.md
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# NATS .NET Porting Status Report
|
||||||
|
|
||||||
|
Generated: 2026-02-27 10:34:31 UTC
|
||||||
|
|
||||||
|
## Modules (12 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| verified | 12 |
|
||||||
|
|
||||||
|
## Features (3673 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| deferred | 2500 |
|
||||||
|
| n_a | 18 |
|
||||||
|
| stub | 168 |
|
||||||
|
| verified | 987 |
|
||||||
|
|
||||||
|
## Unit Tests (3257 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| deferred | 2680 |
|
||||||
|
| n_a | 187 |
|
||||||
|
| verified | 390 |
|
||||||
|
|
||||||
|
## Library Mappings (36 total)
|
||||||
|
|
||||||
|
| Status | Count |
|
||||||
|
|--------|-------|
|
||||||
|
| mapped | 36 |
|
||||||
|
|
||||||
|
|
||||||
|
## Overall Progress
|
||||||
|
|
||||||
|
**1594/6942 items complete (23.0%)**
|
||||||
@@ -7,18 +7,28 @@ namespace NatsNet.PortTracker.Commands;
|
|||||||
|
|
||||||
public static class AuditCommand
|
public static class AuditCommand
|
||||||
{
|
{
|
||||||
|
private record AuditTarget(string Table, string Label, string DefaultSource, string DefaultOutput);
|
||||||
|
|
||||||
|
private static readonly AuditTarget FeaturesTarget = new(
|
||||||
|
"features", "features",
|
||||||
|
Path.Combine(Directory.GetCurrentDirectory(), "dotnet", "src", "ZB.MOM.NatsNet.Server"),
|
||||||
|
Path.Combine(Directory.GetCurrentDirectory(), "reports", "audit-results.csv"));
|
||||||
|
|
||||||
|
private static readonly AuditTarget TestsTarget = new(
|
||||||
|
"unit_tests", "unit tests",
|
||||||
|
Path.Combine(Directory.GetCurrentDirectory(), "dotnet", "tests", "ZB.MOM.NatsNet.Server.Tests"),
|
||||||
|
Path.Combine(Directory.GetCurrentDirectory(), "reports", "audit-results-tests.csv"));
|
||||||
|
|
||||||
public static Command Create(Option<string> dbOption)
|
public static Command Create(Option<string> dbOption)
|
||||||
{
|
{
|
||||||
var sourceOpt = new Option<string>("--source")
|
var sourceOpt = new Option<string?>("--source")
|
||||||
{
|
{
|
||||||
Description = "Path to the .NET source directory",
|
Description = "Path to the .NET source directory (defaults based on --type)"
|
||||||
DefaultValueFactory = _ => Path.Combine(Directory.GetCurrentDirectory(), "dotnet", "src", "ZB.MOM.NatsNet.Server")
|
|
||||||
};
|
};
|
||||||
|
|
||||||
var outputOpt = new Option<string>("--output")
|
var outputOpt = new Option<string?>("--output")
|
||||||
{
|
{
|
||||||
Description = "CSV report output path",
|
Description = "CSV report output path (defaults based on --type)"
|
||||||
DefaultValueFactory = _ => Path.Combine(Directory.GetCurrentDirectory(), "reports", "audit-results.csv")
|
|
||||||
};
|
};
|
||||||
|
|
||||||
var moduleOpt = new Option<int?>("--module")
|
var moduleOpt = new Option<int?>("--module")
|
||||||
@@ -32,44 +42,62 @@ public static class AuditCommand
|
|||||||
DefaultValueFactory = _ => false
|
DefaultValueFactory = _ => false
|
||||||
};
|
};
|
||||||
|
|
||||||
var cmd = new Command("audit", "Classify unknown features by inspecting .NET source code");
|
var typeOpt = new Option<string>("--type")
|
||||||
|
{
|
||||||
|
Description = "What to audit: features, tests, or all",
|
||||||
|
DefaultValueFactory = _ => "features"
|
||||||
|
};
|
||||||
|
|
||||||
|
var cmd = new Command("audit", "Classify unknown features/tests by inspecting .NET source code");
|
||||||
cmd.Add(sourceOpt);
|
cmd.Add(sourceOpt);
|
||||||
cmd.Add(outputOpt);
|
cmd.Add(outputOpt);
|
||||||
cmd.Add(moduleOpt);
|
cmd.Add(moduleOpt);
|
||||||
cmd.Add(executeOpt);
|
cmd.Add(executeOpt);
|
||||||
|
cmd.Add(typeOpt);
|
||||||
|
|
||||||
cmd.SetAction(parseResult =>
|
cmd.SetAction(parseResult =>
|
||||||
{
|
{
|
||||||
var dbPath = parseResult.GetValue(dbOption)!;
|
var dbPath = parseResult.GetValue(dbOption)!;
|
||||||
var sourcePath = parseResult.GetValue(sourceOpt)!;
|
var sourceOverride = parseResult.GetValue(sourceOpt);
|
||||||
var outputPath = parseResult.GetValue(outputOpt)!;
|
var outputOverride = parseResult.GetValue(outputOpt);
|
||||||
var moduleId = parseResult.GetValue(moduleOpt);
|
var moduleId = parseResult.GetValue(moduleOpt);
|
||||||
var execute = parseResult.GetValue(executeOpt);
|
var execute = parseResult.GetValue(executeOpt);
|
||||||
|
var type = parseResult.GetValue(typeOpt)!;
|
||||||
|
|
||||||
RunAudit(dbPath, sourcePath, outputPath, moduleId, execute);
|
AuditTarget[] targets = type switch
|
||||||
|
{
|
||||||
|
"features" => [FeaturesTarget],
|
||||||
|
"tests" => [TestsTarget],
|
||||||
|
"all" => [FeaturesTarget, TestsTarget],
|
||||||
|
_ => throw new ArgumentException($"Unknown audit type: {type}. Use features, tests, or all.")
|
||||||
|
};
|
||||||
|
|
||||||
|
foreach (var target in targets)
|
||||||
|
{
|
||||||
|
var sourcePath = sourceOverride ?? target.DefaultSource;
|
||||||
|
var outputPath = outputOverride ?? target.DefaultOutput;
|
||||||
|
RunAudit(dbPath, sourcePath, outputPath, moduleId, execute, target);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
return cmd;
|
return cmd;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void RunAudit(string dbPath, string sourcePath, string outputPath, int? moduleId, bool execute)
|
private static void RunAudit(string dbPath, string sourcePath, string outputPath, int? moduleId, bool execute, AuditTarget target)
|
||||||
{
|
{
|
||||||
// Validate source directory
|
|
||||||
if (!Directory.Exists(sourcePath))
|
if (!Directory.Exists(sourcePath))
|
||||||
{
|
{
|
||||||
Console.WriteLine($"Error: source directory not found: {sourcePath}");
|
Console.WriteLine($"Error: source directory not found: {sourcePath}");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1. Build source index
|
|
||||||
Console.WriteLine($"Parsing .NET source files in {sourcePath}...");
|
Console.WriteLine($"Parsing .NET source files in {sourcePath}...");
|
||||||
var indexer = new SourceIndexer();
|
var indexer = new SourceIndexer();
|
||||||
indexer.IndexDirectory(sourcePath);
|
indexer.IndexDirectory(sourcePath);
|
||||||
Console.WriteLine($"Indexed {indexer.FilesIndexed} files, {indexer.MethodsIndexed} methods/properties.");
|
Console.WriteLine($"Indexed {indexer.FilesIndexed} files, {indexer.MethodsIndexed} methods/properties.");
|
||||||
|
|
||||||
// 2. Query unknown features
|
|
||||||
using var db = new Database(dbPath);
|
using var db = new Database(dbPath);
|
||||||
var sql = "SELECT id, dotnet_class, dotnet_method, go_file, go_method FROM features WHERE status = 'unknown'";
|
var sql = $"SELECT id, dotnet_class, dotnet_method, go_file, go_method FROM {target.Table} WHERE status = 'unknown'";
|
||||||
var parameters = new List<(string, object?)>();
|
var parameters = new List<(string, object?)>();
|
||||||
if (moduleId is not null)
|
if (moduleId is not null)
|
||||||
{
|
{
|
||||||
@@ -81,12 +109,11 @@ public static class AuditCommand
|
|||||||
var rows = db.Query(sql, parameters.ToArray());
|
var rows = db.Query(sql, parameters.ToArray());
|
||||||
if (rows.Count == 0)
|
if (rows.Count == 0)
|
||||||
{
|
{
|
||||||
Console.WriteLine("No unknown features found.");
|
Console.WriteLine($"No unknown {target.Label} found.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Console.WriteLine($"Found {rows.Count} unknown features to classify.\n");
|
Console.WriteLine($"Found {rows.Count} unknown {target.Label} to classify.\n");
|
||||||
|
|
||||||
// 3. Classify each feature
|
|
||||||
var classifier = new FeatureClassifier(indexer);
|
var classifier = new FeatureClassifier(indexer);
|
||||||
var results = new List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)>();
|
var results = new List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)>();
|
||||||
|
|
||||||
@@ -103,17 +130,16 @@ public static class AuditCommand
|
|||||||
results.Add((feature, result));
|
results.Add((feature, result));
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Write CSV report
|
|
||||||
WriteCsvReport(outputPath, results);
|
WriteCsvReport(outputPath, results);
|
||||||
|
|
||||||
// 5. Print console summary
|
|
||||||
var grouped = results.GroupBy(r => r.Result.Status)
|
var grouped = results.GroupBy(r => r.Result.Status)
|
||||||
.ToDictionary(g => g.Key, g => g.Count());
|
.ToDictionary(g => g.Key, g => g.Count());
|
||||||
|
|
||||||
Console.WriteLine("Feature Status Audit Results");
|
var label = char.ToUpper(target.Label[0]) + target.Label[1..];
|
||||||
Console.WriteLine("=============================");
|
Console.WriteLine($"{label} Status Audit Results");
|
||||||
|
Console.WriteLine(new string('=', $"{label} Status Audit Results".Length));
|
||||||
Console.WriteLine($"Source: {sourcePath} ({indexer.FilesIndexed} files, {indexer.MethodsIndexed} methods indexed)");
|
Console.WriteLine($"Source: {sourcePath} ({indexer.FilesIndexed} files, {indexer.MethodsIndexed} methods indexed)");
|
||||||
Console.WriteLine($"Features audited: {results.Count}");
|
Console.WriteLine($"{label} audited: {results.Count}");
|
||||||
Console.WriteLine();
|
Console.WriteLine();
|
||||||
Console.WriteLine($" verified: {grouped.GetValueOrDefault("verified", 0)}");
|
Console.WriteLine($" verified: {grouped.GetValueOrDefault("verified", 0)}");
|
||||||
Console.WriteLine($" stub: {grouped.GetValueOrDefault("stub", 0)}");
|
Console.WriteLine($" stub: {grouped.GetValueOrDefault("stub", 0)}");
|
||||||
@@ -128,8 +154,7 @@ public static class AuditCommand
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 6. Apply DB updates
|
ApplyUpdates(db, results, target);
|
||||||
ApplyUpdates(db, results);
|
|
||||||
Console.WriteLine($"Report: {outputPath}");
|
Console.WriteLine($"Report: {outputPath}");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,7 +162,6 @@ public static class AuditCommand
|
|||||||
string outputPath,
|
string outputPath,
|
||||||
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results)
|
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results)
|
||||||
{
|
{
|
||||||
// Ensure directory exists
|
|
||||||
var dir = Path.GetDirectoryName(outputPath);
|
var dir = Path.GetDirectoryName(outputPath);
|
||||||
if (!string.IsNullOrEmpty(dir))
|
if (!string.IsNullOrEmpty(dir))
|
||||||
Directory.CreateDirectory(dir);
|
Directory.CreateDirectory(dir);
|
||||||
@@ -153,9 +177,9 @@ public static class AuditCommand
|
|||||||
|
|
||||||
private static void ApplyUpdates(
|
private static void ApplyUpdates(
|
||||||
Database db,
|
Database db,
|
||||||
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results)
|
List<(FeatureClassifier.FeatureRecord Feature, FeatureClassifier.ClassificationResult Result)> results,
|
||||||
|
AuditTarget target)
|
||||||
{
|
{
|
||||||
// Group by (status, notes) for efficient batch updates
|
|
||||||
var groups = results
|
var groups = results
|
||||||
.GroupBy(r => (r.Result.Status, Notes: r.Result.Status == "n_a" ? r.Result.Reason : (string?)null))
|
.GroupBy(r => (r.Result.Status, Notes: r.Result.Status == "n_a" ? r.Result.Reason : (string?)null))
|
||||||
.ToList();
|
.ToList();
|
||||||
@@ -170,7 +194,6 @@ public static class AuditCommand
|
|||||||
var status = group.Key.Status;
|
var status = group.Key.Status;
|
||||||
var notes = group.Key.Notes;
|
var notes = group.Key.Notes;
|
||||||
|
|
||||||
// Build parameterized IN clause
|
|
||||||
var placeholders = new List<string>();
|
var placeholders = new List<string>();
|
||||||
using var cmd = db.CreateCommand("");
|
using var cmd = db.CreateCommand("");
|
||||||
for (var i = 0; i < ids.Count; i++)
|
for (var i = 0; i < ids.Count; i++)
|
||||||
@@ -183,12 +206,12 @@ public static class AuditCommand
|
|||||||
|
|
||||||
if (notes is not null)
|
if (notes is not null)
|
||||||
{
|
{
|
||||||
cmd.CommandText = $"UPDATE features SET status = @status, notes = @notes WHERE id IN ({string.Join(", ", placeholders)})";
|
cmd.CommandText = $"UPDATE {target.Table} SET status = @status, notes = @notes WHERE id IN ({string.Join(", ", placeholders)})";
|
||||||
cmd.Parameters.AddWithValue("@notes", notes);
|
cmd.Parameters.AddWithValue("@notes", notes);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
cmd.CommandText = $"UPDATE features SET status = @status WHERE id IN ({string.Join(", ", placeholders)})";
|
cmd.CommandText = $"UPDATE {target.Table} SET status = @status WHERE id IN ({string.Join(", ", placeholders)})";
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.Transaction = transaction;
|
cmd.Transaction = transaction;
|
||||||
@@ -197,7 +220,7 @@ public static class AuditCommand
|
|||||||
}
|
}
|
||||||
|
|
||||||
transaction.Commit();
|
transaction.Commit();
|
||||||
Console.WriteLine($"Updated {totalUpdated} features.");
|
Console.WriteLine($"Updated {totalUpdated} {target.Label}.");
|
||||||
}
|
}
|
||||||
catch
|
catch
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user