Issue #49: add cross-language smoke matrix

2026-04-26 21:21:49 -04:00
parent 9159f6f093
commit 79f73e04fd
4 changed files with 662 additions and 0 deletions
@@ -0,0 +1,276 @@
+using System.Text.Json;
+
+namespace MxGateway.Tests.Contracts;
+
+public sealed class CrossLanguageSmokeMatrixTests
+{
+    [Fact]
+    public void Matrix_DeclaresIntegrationGateAndComparisonShape()
+    {
+        using JsonDocument matrix = LoadSmokeMatrix();
+        JsonElement root = matrix.RootElement;
+
+        Assert.Equal(1, root.GetProperty("schemaVersion").GetInt32());
+        Assert.Equal("mxaccess-gateway-cross-language-smoke-matrix", root.GetProperty("fixtureSet").GetString());
+
+        JsonElement integrationGate = root.GetProperty("integrationGate");
+        Assert.Equal("MXGATEWAY_INTEGRATION", integrationGate.GetProperty("variable").GetString());
+        Assert.Equal("1", integrationGate.GetProperty("requiredValue").GetString());
+
+        JsonElement defaultInputs = root.GetProperty("defaultInputs");
+        Assert.Equal("MXGATEWAY_ENDPOINT", defaultInputs.GetProperty("endpointVariable").GetString());
+        Assert.Equal("localhost:5000", defaultInputs.GetProperty("endpointFallback").GetString());
+        Assert.Equal("MXGATEWAY_API_KEY", defaultInputs.GetProperty("apiKeyVariable").GetString());
+        Assert.Equal("MXGATEWAY_TEST_ITEM", defaultInputs.GetProperty("itemVariable").GetString());
+
+        AssertRequiredFields(
+            root.GetProperty("jsonComparison").GetProperty("commonFields"),
+            "language",
+            "operation",
+            "sessionId",
+            "serverHandle",
+            "itemHandle",
+            "events",
+            "closeStatus");
+        AssertRequiredFields(
+            root.GetProperty("failureOutput").GetProperty("requiredContextFields"),
+            "language",
+            "endpoint",
+            "authContext");
+
+        JsonElement authContext = root.GetProperty("failureOutput").GetProperty("authContext");
+        Assert.Equal("MXGATEWAY_API_KEY", authContext.GetProperty("sourceVariable").GetString());
+        Assert.Equal("<redacted>", authContext.GetProperty("redactedValue").GetString());
+        AssertForbiddenLiterals(authContext.GetProperty("forbiddenLiterals"));
+    }
+
+    [Fact]
+    public void Matrix_CoversEverySupportedClientWithEquivalentSmokeSteps()
+    {
+        using JsonDocument matrix = LoadSmokeMatrix();
+        JsonElement root = matrix.RootElement;
+        string[] requiredOperations = GetStrings(root.GetProperty("requiredOperations"));
+        Dictionary<string, JsonElement> clientsByLanguage = [];
+
+        foreach (JsonElement client in root.GetProperty("clients").EnumerateArray())
+        {
+            string language = client.GetProperty("language").GetString()!;
+
+            Assert.True(clientsByLanguage.TryAdd(language, client), $"Duplicate smoke client '{language}'.");
+            Assert.Contains(language, ExpectedLanguages);
+            AssertClientWorkDirectoryExists(client);
+            AssertIntegrationSkip(client);
+            AssertRequiredFields(client.GetProperty("failureContextFields"), "language", "endpoint", "authContext");
+            AssertSmokeCommands(client, requiredOperations);
+            AssertOptionalWriteCommand(client.GetProperty("optionalWriteCommand").GetString()!);
+            AssertCommandUsesJsonAndAuthEnv(client.GetProperty("bundledSmokeCommand").GetString()!);
+            Assert.Contains("TestChildObject.TestInt", client.GetProperty("bundledSmokeCommand").GetString()!, StringComparison.Ordinal);
+        }
+
+        Assert.Equal(ExpectedLanguages.OrderBy(language => language, StringComparer.Ordinal), clientsByLanguage.Keys.OrderBy(language => language, StringComparer.Ordinal));
+    }
+
+    [Fact]
+    public void Matrix_KeepsLiveSmokeOptInAndSecretsOutOfCommands()
+    {
+        using JsonDocument matrix = LoadSmokeMatrix();
+        JsonElement root = matrix.RootElement;
+        string[] forbiddenLiterals = GetStrings(root.GetProperty("failureOutput").GetProperty("authContext").GetProperty("forbiddenLiterals"));
+
+        foreach (JsonElement client in root.GetProperty("clients").EnumerateArray())
+        {
+            string language = client.GetProperty("language").GetString()!;
+
+            AssertIntegrationSkip(client);
+
+            foreach (JsonElement commandStep in client.GetProperty("commands").EnumerateArray())
+            {
+                AssertNoForbiddenLiterals(language, commandStep.GetProperty("command").GetString()!, forbiddenLiterals);
+            }
+
+            AssertNoForbiddenLiterals(language, client.GetProperty("optionalWriteCommand").GetString()!, forbiddenLiterals);
+            AssertNoForbiddenLiterals(language, client.GetProperty("bundledSmokeCommand").GetString()!, forbiddenLiterals);
+        }
+    }
+
+    private static readonly string[] ExpectedLanguages =
+    [
+        "dotnet",
+        "go",
+        "rust",
+        "python",
+        "java",
+    ];
+
+    private static void AssertSmokeCommands(
+        JsonElement client,
+        string[] requiredOperations)
+    {
+        Dictionary<string, JsonElement> commandsByOperation = [];
+
+        foreach (JsonElement commandStep in client.GetProperty("commands").EnumerateArray())
+        {
+            string operation = commandStep.GetProperty("operation").GetString()!;
+            string command = commandStep.GetProperty("command").GetString()!;
+
+            Assert.True(commandsByOperation.TryAdd(operation, commandStep), $"Duplicate smoke operation '{operation}'.");
+            AssertCommandUsesJsonAndAuthEnv(command);
+            Assert.Contains("localhost:5000", command, StringComparison.Ordinal);
+            AssertOperationPlaceholders(operation, command);
+        }
+
+        Assert.Equal(requiredOperations.OrderBy(operation => operation, StringComparer.Ordinal), commandsByOperation.Keys.OrderBy(operation => operation, StringComparer.Ordinal));
+    }
+
+    private static void AssertOperationPlaceholders(
+        string operation,
+        string command)
+    {
+        switch (operation)
+        {
+            case "open-session":
+                Assert.Contains("smoke", command, StringComparison.Ordinal);
+                break;
+            case "register":
+                Assert.Contains("<session-id>", command, StringComparison.Ordinal);
+                Assert.Contains("smoke", command, StringComparison.Ordinal);
+                break;
+            case "add-item":
+                Assert.Contains("<session-id>", command, StringComparison.Ordinal);
+                Assert.Contains("<server-handle>", command, StringComparison.Ordinal);
+                Assert.Contains("TestChildObject.TestInt", command, StringComparison.Ordinal);
+                break;
+            case "advise":
+                Assert.Contains("<session-id>", command, StringComparison.Ordinal);
+                Assert.Contains("<server-handle>", command, StringComparison.Ordinal);
+                Assert.Contains("<item-handle>", command, StringComparison.Ordinal);
+                break;
+            case "stream-events":
+                Assert.Contains("<session-id>", command, StringComparison.Ordinal);
+                Assert.True(
+                    command.Contains("--max-events 1", StringComparison.Ordinal)
+                    || command.Contains("-limit 1", StringComparison.Ordinal)
+                    || command.Contains("--limit 1", StringComparison.Ordinal),
+                    $"Stream command '{command}' must bound event reads.");
+                break;
+            case "close-session":
+                Assert.Contains("<session-id>", command, StringComparison.Ordinal);
+                break;
+            default:
+                throw new InvalidOperationException($"Unexpected smoke operation '{operation}'.");
+        }
+    }
+
+    private static void AssertOptionalWriteCommand(string command)
+    {
+        AssertCommandUsesJsonAndAuthEnv(command);
+        Assert.Contains("<session-id>", command, StringComparison.Ordinal);
+        Assert.Contains("<server-handle>", command, StringComparison.Ordinal);
+        Assert.Contains("<item-handle>", command, StringComparison.Ordinal);
+        Assert.Contains("<write-value>", command, StringComparison.Ordinal);
+        Assert.Contains("int32", command, StringComparison.Ordinal);
+    }
+
+    private static void AssertCommandUsesJsonAndAuthEnv(string command)
+    {
+        Assert.True(
+            command.Contains("--json", StringComparison.Ordinal) || command.Contains("-json", StringComparison.Ordinal),
+            $"Command '{command}' must request JSON output.");
+        Assert.Contains("MXGATEWAY_API_KEY", command, StringComparison.Ordinal);
+        Assert.Contains("api-key-env", command, StringComparison.Ordinal);
+    }
+
+    private static void AssertIntegrationSkip(JsonElement client)
+    {
+        JsonElement integrationSkip = client.GetProperty("integrationSkip");
+
+        Assert.Equal("MXGATEWAY_INTEGRATION", integrationSkip.GetProperty("variable").GetString());
+        Assert.Equal("1", integrationSkip.GetProperty("requiredValue").GetString());
+    }
+
+    private static void AssertClientWorkDirectoryExists(JsonElement client)
+    {
+        string workingDirectory = client.GetProperty("workingDirectory").GetString()!;
+        DirectoryInfo repositoryRoot = FindRepositoryRoot();
+        string fullPath = Path.GetFullPath(Path.Combine(repositoryRoot.FullName, workingDirectory));
+
+        Assert.True(Directory.Exists(fullPath), $"Smoke client working directory '{workingDirectory}' must exist.");
+        Assert.StartsWith(repositoryRoot.FullName, fullPath, StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static void AssertRequiredFields(
+        JsonElement fields,
+        params string[] expectedFields)
+    {
+        HashSet<string> declared = GetStrings(fields).ToHashSet(StringComparer.Ordinal);
+
+        foreach (string expectedField in expectedFields)
+        {
+            Assert.Contains(expectedField, declared);
+        }
+    }
+
+    private static void AssertForbiddenLiterals(JsonElement forbiddenLiterals)
+    {
+        string[] values = GetStrings(forbiddenLiterals);
+
+        Assert.Contains("mxgw_visible_secret", values);
+        Assert.Contains("Bearer mxgw_visible_secret", values);
+    }
+
+    private static void AssertNoForbiddenLiterals(
+        string language,
+        string command,
+        string[] forbiddenLiterals)
+    {
+        foreach (string forbiddenLiteral in forbiddenLiterals)
+        {
+            Assert.DoesNotContain(forbiddenLiteral, command, StringComparison.Ordinal);
+        }
+
+        Assert.DoesNotContain(" --api-key ", command, StringComparison.Ordinal);
+        Assert.DoesNotContain(" -api-key ", command, StringComparison.Ordinal);
+        Assert.Contains("api-key-env", command, StringComparison.Ordinal);
+        Assert.Contains("MXGATEWAY_API_KEY", command, StringComparison.Ordinal);
+        Assert.False(command.Contains("Bearer ", StringComparison.Ordinal), $"Smoke command for '{language}' must not include bearer tokens.");
+    }
+
+    private static string[] GetStrings(JsonElement array)
+    {
+        return array
+            .EnumerateArray()
+            .Select(element => element.GetString()!)
+            .ToArray();
+    }
+
+    private static JsonDocument LoadSmokeMatrix()
+    {
+        return JsonDocument.Parse(File.ReadAllText(Path.Combine(GetSmokeFixtureRoot().FullName, "cross-language-smoke-matrix.json")));
+    }
+
+    private static DirectoryInfo GetSmokeFixtureRoot()
+    {
+        DirectoryInfo repositoryRoot = FindRepositoryRoot();
+
+        return new DirectoryInfo(Path.Combine(repositoryRoot.FullName, "clients", "proto", "fixtures", "smoke"));
+    }
+
+    private static DirectoryInfo FindRepositoryRoot()
+    {
+        DirectoryInfo? current = new(AppContext.BaseDirectory);
+
+        while (current is not null)
+        {
+            if (File.Exists(Path.Combine(current.FullName, "AGENTS.md"))
+                && Directory.Exists(Path.Combine(current.FullName, "src"))
+                && Directory.Exists(Path.Combine(current.FullName, "clients")))
+            {
+                return current;
+            }
+
+            current = current.Parent;
+        }
+
+        throw new DirectoryNotFoundException("Could not locate the repository root from the test output directory.");
+    }
+}