mxaccessgw/scripts/run-client-e2e-tests.ps1

<#
.SYNOPSIS
Cross-language client e2e matrix for the MXAccess Gateway.

.DESCRIPTION
Drives the .NET, Go, Rust, Python, and Java client CLIs against a running
gateway + worker. For each language the script exercises session open/close,
register, bulk subscribe/unsubscribe, per-tag add-item/advise, event
streaming, a write round-trip with value assertion, error-path (parity)
checks, and API-key auth rejection.

The gateway and worker are assumed to be already running at -Endpoint; the
script does not start or stop them.
#>
[CmdletBinding()]
param(
    [string[]]$Clients = @("dotnet", "go", "rust", "python", "java"),
    [int]$MachineStart = 1,
    [int]$MachineEnd = 20,
    [string[]]$Attributes = @(
        "ProtectedValue",
        "TestChangingInt",
        "TestBoolArray",
        "TestIntArray",
        "TestDateTimeArray",
        "TestStringArray"
    ),
    [string]$Endpoint = "localhost:5000",
    [string]$ApiKeyEnv = "MXGATEWAY_API_KEY",
    [string]$SqlServer = "localhost",
    [string]$Database = "ZB",
    [int]$EventLimit = 5,
    [int]$BulkTagCount = 6,
    [switch]$SkipStream,
    [switch]$SkipBulk,
    # Write round-trip + value assertion.
    [switch]$SkipWrite,
    [string]$WriteAttribute = "TestChangingInt",
    [string]$WriteType = "int32",
    [int]$WriteValueBase = 424200,
    [int]$WriteEchoMaxEvents = 200,
    # Error-path (parity) checks.
    [switch]$SkipParity,
    # API-key auth rejection checks.
    [switch]$SkipAuth,
    # Optional env var holding an API key whose scopes are insufficient for
    # open-session; when supplied the auth phase also asserts that key is
    # rejected (PermissionDenied) on top of the always-on missing-key check.
    [string]$RejectScopeApiKeyEnv,
    # Run each language client concurrently as an isolated child process.
    [switch]$Parallel,
    [switch]$DryRun,
    [string]$ReportPath,
    # Internal: set by -Parallel on each spawned child so it always writes its
    # report (even under -DryRun) for the parent to merge. Not for direct use.
    [switch]$EmitReport
)

Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"

$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
$discoveryScript = Join-Path $PSScriptRoot "discover-testmachine-tags.ps1"
$validClients = @("dotnet", "go", "rust", "python", "java")
$Clients = @($Clients | ForEach-Object {
    $_ -split ","
} | ForEach-Object {
    $_.Trim().ToLowerInvariant()
} | Where-Object {
    -not [string]::IsNullOrWhiteSpace($_)
})
$Attributes = @($Attributes | ForEach-Object {
    $_ -split ","
} | ForEach-Object {
    $_.Trim()
} | Where-Object {
    -not [string]::IsNullOrWhiteSpace($_)
})

if ($Clients.Count -eq 0) {
    throw "At least one client is required."
}

if ($Attributes.Count -eq 0) {
    throw "At least one attribute is required."
}

if ($BulkTagCount -lt 1) {
    throw "BulkTagCount must be greater than zero."
}

if ($WriteEchoMaxEvents -lt 1) {
    throw "WriteEchoMaxEvents must be greater than zero."
}

foreach ($client in $Clients) {
    if ($validClients -notcontains $client) {
        throw "Unsupported client '$client'. Supported clients: $($validClients -join ', ')."
    }
}

if ([string]::IsNullOrWhiteSpace($ReportPath)) {
    $timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
    $ReportPath = Join-Path $repoRoot "artifacts/e2e/testmachine-client-e2e-$timestamp.json"
}

function ConvertTo-HttpEndpoint {
    param([string]$Value)

    if ($Value.StartsWith("http://", [StringComparison]::OrdinalIgnoreCase) -or
        $Value.StartsWith("https://", [StringComparison]::OrdinalIgnoreCase)) {
        return $Value
    }

    return "http://$Value"
}

function ConvertTo-HostEndpoint {
    param([string]$Value)

    $hostValue = $Value
    if ($hostValue.StartsWith("http://", [StringComparison]::OrdinalIgnoreCase)) {
        $hostValue = $hostValue.Substring(7)
    } elseif ($hostValue.StartsWith("https://", [StringComparison]::OrdinalIgnoreCase)) {
        $hostValue = $hostValue.Substring(8)
    }

    return $hostValue.TrimEnd("/")
}

function Join-CommandLine {
    param(
        [string]$FilePath,
        [string[]]$Arguments
    )

    $parts = @($FilePath) + $Arguments
    return ($parts | ForEach-Object { ConvertTo-NativeArgument -Value $_ }) -join " "
}

function ConvertTo-NativeArgument {
    param([string]$Value)

    if ($Value -notmatch '[\s"]') {
        return $Value
    }

    return '"' + $Value.Replace('\', '\\').Replace('"', '\"') + '"'
}

function Invoke-NativeCommand {
    param(
        [string]$FilePath,
        [string[]]$Arguments,
        [string]$WorkingDirectory,
        [hashtable]$Environment = @{},
        # When set, a non-zero exit code is returned to the caller instead of
        # throwing. Used by the parity and auth phases, which expect failure.
        [switch]$AllowFailure
    )

    $process = [System.Diagnostics.Process]::new()
    $process.StartInfo.FileName = $FilePath
    $process.StartInfo.Arguments = ($Arguments | ForEach-Object {
        ConvertTo-NativeArgument -Value $_
    }) -join " "
    $process.StartInfo.WorkingDirectory = $WorkingDirectory
    $process.StartInfo.RedirectStandardOutput = $true
    $process.StartInfo.RedirectStandardError = $true
    $process.StartInfo.UseShellExecute = $false
    foreach ($entry in $Environment.GetEnumerator()) {
        $process.StartInfo.Environment[$entry.Key] = [string]$entry.Value
    }

    $commandLine = Join-CommandLine -FilePath $FilePath -Arguments $Arguments
    if ($DryRun) {
        Write-Host "[dry-run] $commandLine"
        return [pscustomobject]@{
            commandLine = $commandLine
            exitCode = 0
            stdout = "{}"
            stderr = ""
        }
    }

    [void]$process.Start()
    $stdout = $process.StandardOutput.ReadToEnd()
    $stderr = $process.StandardError.ReadToEnd()
    $process.WaitForExit()

    $result = [pscustomobject]@{
        commandLine = $commandLine
        exitCode = $process.ExitCode
        stdout = $stdout
        stderr = $stderr
    }

    if ($result.exitCode -ne 0 -and -not $AllowFailure) {
        throw "Command failed with exit code $($result.exitCode): $commandLine`n$stderr`n$stdout"
    }

    return $result
}

function Read-JsonObject {
    param([string]$Text)

    if ([string]::IsNullOrWhiteSpace($Text)) {
        return $null
    }

    try {
        return $Text | ConvertFrom-Json
    } catch {
        $jsonLines = @()
        foreach ($line in ($Text -split "`r?`n")) {
            $trimmed = $line.Trim()
            if ($trimmed.StartsWith("{") -and $trimmed.EndsWith("}")) {
                $jsonLines += ($trimmed | ConvertFrom-Json)
            }
        }
        return $jsonLines
    }
}

function Get-OpenSessionId {
    param(
        [string]$Client,
        [object]$Json
    )

    switch ($Client) {
        "dotnet" { return $Json.sessionId }
        "go" { return $Json.reply.sessionId }
        "rust" { return $Json.sessionId }
        "python" { return $Json.sessionId }
        "java" { return $Json.reply.sessionId }
    }
}

function Get-ServerHandle {
    param(
        [string]$Client,
        [object]$Json
    )

    switch ($Client) {
        "dotnet" { return [int]$Json.register.serverHandle }
        "go" { return [int]$Json.reply.register.serverHandle }
        "rust" { return [int]$Json.serverHandle }
        "python" { return [int]$Json.serverHandle }
        "java" { return [int]$Json.reply.register.serverHandle }
    }
}

function Get-ItemHandle {
    param(
        [string]$Client,
        [object]$Json
    )

    switch ($Client) {
        "dotnet" { return [int]$Json.addItem.itemHandle }
        "go" { return [int]$Json.reply.addItem.itemHandle }
        "rust" { return [int]$Json.itemHandle }
        "python" { return [int]$Json.itemHandle }
        "java" { return [int]$Json.reply.addItem.itemHandle }
    }
}

function Get-StreamEventCount {
    param(
        [string]$Client,
        [object]$Json
    )

    switch ($Client) {
        "dotnet" { return @($Json.events).Count }
        "go" { return @($Json).Count }
        "rust" { return [int]$Json.eventCount }
        "python" { return @($Json.events).Count }
        "java" { return @($Json).Count }
    }
}

# Returns the per-event objects from a stream-events reply so the write
# round-trip can inspect their values. Mirrors Get-StreamEventCount: .NET,
# Rust, and Python aggregate under `events`; Go and Java emit one event
# object per line (Read-JsonObject collapses NDJSON into an array).
function Get-StreamEvents {
    param(
        [string]$Client,
        [object]$Json
    )

    switch ($Client) {
        "dotnet" { return @($Json.events) }
        "go" { return @($Json) }
        "rust" { return @($Json.events) }
        "python" { return @($Json.events) }
        "java" { return @($Json) }
    }
}

function Get-PropertyValue {
    param(
        [object]$Object,
        [string[]]$Names
    )

    if ($null -eq $Object) {
        return $null
    }

    foreach ($name in $Names) {
        $property = $Object.PSObject.Properties[$name]
        if ($null -ne $property) {
            return $property.Value
        }
    }

    return $null
}

# Extracts the item handle from a streamed event, tolerating camelCase
# (protobuf-JSON) and snake_case (some MessageToDict shapes).
function Get-EventItemHandle {
    param([object]$Event)

    $handle = Get-PropertyValue -Object $Event -Names @("itemHandle", "item_handle")
    if ($null -eq $handle) {
        return $null
    }

    return [int]$handle
}

# Extracts the scalar payload from a streamed event's MxValue as a string.
# The MxValue oneof renders to one protobuf-JSON `*Value` key; all five
# CLIs (after the Rust stream-events extension) emit the same key names.
function Get-EventScalar {
    param([object]$Event)

    $value = Get-PropertyValue -Object $Event -Names @("value")
    if ($null -eq $value) {
        return $null
    }

    foreach ($key in @("boolValue", "int32Value", "int64Value", "floatValue", "doubleValue", "stringValue")) {
        $property = $value.PSObject.Properties[$key]
        if ($null -ne $property -and $null -ne $property.Value) {
            return [string]$property.Value
        }
    }

    return $null
}

# Compares a written value against an observed event scalar. Numeric values
# are compared numerically (so 42 matches 42.0); everything else compares as
# a trimmed, case-insensitive string.
function Test-ValueEquals {
    param(
        [string]$Expected,
        [string]$Observed
    )

    if ([string]::IsNullOrWhiteSpace($Observed)) {
        return $false
    }

    $expectedNumber = 0.0
    $observedNumber = 0.0
    if ([double]::TryParse($Expected, [ref]$expectedNumber) -and
        [double]::TryParse($Observed, [ref]$observedNumber)) {
        return $expectedNumber -eq $observedNumber
    }

    return [string]::Equals($Expected.Trim(), $Observed.Trim(), [StringComparison]::OrdinalIgnoreCase)
}

function Get-BulkResults {
    param(
        [string]$Client,
        [string]$Operation,
        [object]$Json
    )

    if ($Client -in @("go", "rust", "python", "java")) {
        return @(Get-PropertyValue -Object $Json -Names @("results"))
    }

    $replyName = if ($Operation -eq "subscribe-bulk") { "subscribeBulk" } else { "unsubscribeBulk" }
    $reply = Get-PropertyValue -Object $Json -Names @($replyName)
    return @(Get-PropertyValue -Object $reply -Names @("results"))
}

function Get-BulkItemHandles {
    param([object[]]$Results)

    return @($Results | ForEach-Object {
        [int](Get-PropertyValue -Object $_ -Names @("itemHandle", "item_handle"))
    } | Where-Object {
        $_ -gt 0
    })
}

function Assert-BulkResults {
    param(
        [string]$Client,
        [string]$Operation,
        [object[]]$Results,
        [int]$ExpectedCount
    )

    if ($Results.Count -ne $ExpectedCount) {
        throw "$Client $Operation returned $($Results.Count) result(s); expected $ExpectedCount."
    }

    foreach ($result in $Results) {
        $success = Get-PropertyValue -Object $result -Names @("wasSuccessful", "was_successful")
        if ($null -ne $success -and -not [bool]$success) {
            $tagAddress = Get-PropertyValue -Object $result -Names @("tagAddress", "tag_address")
            $errorMessage = Get-PropertyValue -Object $result -Names @("errorMessage", "error_message")
            throw "$Client $Operation failed for '$tagAddress': $errorMessage"
        }
    }
}

function Get-ClientCommand {
    param(
        [string]$Client,
        [string]$Operation,
        [hashtable]$Values,
        # The environment variable the client reads the API key from. Defaults
        # to the run-wide -ApiKeyEnv; the auth phase overrides it to drive a
        # missing-key or insufficient-scope rejection.
        [string]$ApiKeyEnvName = $ApiKeyEnv
    )

    $httpEndpoint = ConvertTo-HttpEndpoint -Value $Endpoint
    $hostEndpoint = ConvertTo-HostEndpoint -Value $Endpoint
    $clientName = "mxgw-$Client-e2e"
    $streamMaxEvents = if ($Values.ContainsKey("maxEvents")) { [int]$Values.maxEvents } else { $EventLimit }
    # Python's stream-events call ends on a wall-clock timeout; give it enough
    # headroom to drain a large write-echo budget.
    $pythonStreamTimeout = [Math]::Max(15, [int][Math]::Ceiling($streamMaxEvents / 4.0))

    switch ($Client) {
        "dotnet" {
            $arguments = @(
                "run", "--project", "clients/dotnet/MxGateway.Client.Cli", "--",
                $Operation,
                "--endpoint", $httpEndpoint,
                "--api-key-env", $ApiKeyEnvName,
                "--timeout", "60s",
                "--json"
            )
            if ($Operation -eq "open-session") {
                $arguments += @("--client-name", $clientName)
            } elseif ($Operation -eq "register") {
                $arguments += @("--session-id", $Values.sessionId, "--client-name", $clientName)
            } elseif ($Operation -eq "add-item") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item", $Values.item)
            } elseif ($Operation -eq "advise") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handle", "$($Values.itemHandle)")
            } elseif ($Operation -eq "subscribe-bulk") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--items", $Values.items)
            } elseif ($Operation -eq "unsubscribe-bulk") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handles", $Values.itemHandles)
            } elseif ($Operation -eq "write") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handle", "$($Values.itemHandle)", "--type", $Values.valueType, "--value", $Values.value)
            } elseif ($Operation -eq "stream-events") {
                $arguments += @("--session-id", $Values.sessionId, "--max-events", "$streamMaxEvents")
            } elseif ($Operation -eq "close-session") {
                $arguments += @("--session-id", $Values.sessionId)
            }
            return [pscustomobject]@{ file = "dotnet"; args = $arguments; cwd = $repoRoot; env = @{} }
        }
        "go" {
            $arguments = @(
                "run", "./cmd/mxgw-go", $Operation,
                "-endpoint", $hostEndpoint,
                "-api-key-env", $ApiKeyEnvName,
                "-plaintext",
                "-json"
            )
            if ($Operation -eq "open-session") {
                $arguments += @("-client-session-name", $clientName)
            } elseif ($Operation -eq "register") {
                $arguments += @("-session-id", $Values.sessionId, "-client-name", $clientName)
            } elseif ($Operation -eq "add-item") {
                $arguments += @("-session-id", $Values.sessionId, "-server-handle", "$($Values.serverHandle)", "-item", $Values.item)
            } elseif ($Operation -eq "advise") {
                $arguments += @("-session-id", $Values.sessionId, "-server-handle", "$($Values.serverHandle)", "-item-handle", "$($Values.itemHandle)")
            } elseif ($Operation -eq "subscribe-bulk") {
                $arguments += @("-session-id", $Values.sessionId, "-server-handle", "$($Values.serverHandle)", "-items", $Values.items)
            } elseif ($Operation -eq "unsubscribe-bulk") {
                $arguments += @("-session-id", $Values.sessionId, "-server-handle", "$($Values.serverHandle)", "-item-handles", $Values.itemHandles)
            } elseif ($Operation -eq "write") {
                $arguments += @("-session-id", $Values.sessionId, "-server-handle", "$($Values.serverHandle)", "-item-handle", "$($Values.itemHandle)", "-type", $Values.valueType, "-value", $Values.value)
            } elseif ($Operation -eq "stream-events") {
                $arguments += @("-session-id", $Values.sessionId, "-limit", "$streamMaxEvents")
            } elseif ($Operation -eq "close-session") {
                $arguments += @("-session-id", $Values.sessionId)
            }
            return [pscustomobject]@{ file = "go"; args = $arguments; cwd = (Join-Path $repoRoot "clients/go"); env = @{} }
        }
        "rust" {
            $arguments = @(
                "run", "-p", "mxgw-cli", "--", $Operation,
                "--endpoint", $httpEndpoint,
                "--api-key-env", $ApiKeyEnvName,
                "--json"
            )
            if ($Operation -eq "open-session") {
                $arguments += @("--client-name", $clientName)
            } elseif ($Operation -eq "register") {
                $arguments += @("--session-id", $Values.sessionId, "--client-name", $clientName)
            } elseif ($Operation -eq "add-item") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item", $Values.item)
            } elseif ($Operation -eq "advise") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handle", "$($Values.itemHandle)")
            } elseif ($Operation -eq "subscribe-bulk") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--items", $Values.items)
            } elseif ($Operation -eq "unsubscribe-bulk") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handles", $Values.itemHandles)
            } elseif ($Operation -eq "write") {
                # Rust names the type flag --value-type, unlike the other CLIs.
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handle", "$($Values.itemHandle)", "--value-type", $Values.valueType, "--value", $Values.value)
            } elseif ($Operation -eq "stream-events") {
                $arguments += @("--session-id", $Values.sessionId, "--max-events", "$streamMaxEvents")
            } elseif ($Operation -eq "close-session") {
                $arguments += @("--session-id", $Values.sessionId)
            }
            return [pscustomobject]@{ file = "cargo"; args = $arguments; cwd = (Join-Path $repoRoot "clients/rust"); env = @{} }
        }
        "python" {
            $arguments = @(
                "-m", "mxgateway_cli", $Operation,
                "--endpoint", $hostEndpoint,
                "--api-key-env", $ApiKeyEnvName,
                "--plaintext",
                "--json"
            )
            if ($Operation -eq "open-session") {
                $arguments += @("--client-name", $clientName)
            } elseif ($Operation -eq "register") {
                $arguments += @("--session-id", $Values.sessionId, "--client-name", $clientName)
            } elseif ($Operation -eq "add-item") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item", $Values.item)
            } elseif ($Operation -eq "advise") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handle", "$($Values.itemHandle)")
            } elseif ($Operation -eq "subscribe-bulk") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--items", $Values.items)
            } elseif ($Operation -eq "unsubscribe-bulk") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handles", $Values.itemHandles)
            } elseif ($Operation -eq "write") {
                $arguments += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handle", "$($Values.itemHandle)", "--type", $Values.valueType, "--value", $Values.value)
            } elseif ($Operation -eq "stream-events") {
                $arguments += @("--session-id", $Values.sessionId, "--max-events", "$streamMaxEvents", "--timeout", "$pythonStreamTimeout")
            } elseif ($Operation -eq "close-session") {
                $arguments += @("--session-id", $Values.sessionId)
            }
            $env = @{
                PYTHONPATH = Join-Path $repoRoot "clients/python/src"
            }
            return [pscustomobject]@{ file = "py"; args = @("-3.12") + $arguments; cwd = (Join-Path $repoRoot "clients/python"); env = $env }
        }
        "java" {
            $cliArgs = @(
                $Operation,
                "--endpoint", $hostEndpoint,
                "--api-key-env", $ApiKeyEnvName,
                "--plaintext",
                "--json"
            )
            if ($Operation -eq "open-session") {
                $cliArgs += @("--client-session-name", $clientName)
            } elseif ($Operation -eq "register") {
                $cliArgs += @("--session-id", $Values.sessionId, "--client-name", $clientName)
            } elseif ($Operation -eq "add-item") {
                $cliArgs += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item", $Values.item)
            } elseif ($Operation -eq "advise") {
                $cliArgs += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handle", "$($Values.itemHandle)")
            } elseif ($Operation -eq "subscribe-bulk") {
                $cliArgs += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--items", $Values.items)
            } elseif ($Operation -eq "unsubscribe-bulk") {
                $cliArgs += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handles", $Values.itemHandles)
            } elseif ($Operation -eq "write") {
                $cliArgs += @("--session-id", $Values.sessionId, "--server-handle", "$($Values.serverHandle)", "--item-handle", "$($Values.itemHandle)", "--type", $Values.valueType, "--value", $Values.value)
            } elseif ($Operation -eq "stream-events") {
                $cliArgs += @("--session-id", $Values.sessionId, "--limit", "$streamMaxEvents")
            } elseif ($Operation -eq "close-session") {
                $cliArgs += @("--session-id", $Values.sessionId)
            }
            $arguments = @("--quiet", ":mxgateway-cli:run", "--args=$($cliArgs -join ' ')")
            return [pscustomobject]@{ file = "gradle"; args = $arguments; cwd = (Join-Path $repoRoot "clients/java"); env = @{} }
        }
    }
}

# Synthesizes a dry-run JSON reply for an operation so the flow can be
# validated without a live gateway.
function Get-DryRunReply {
    param(
        [string]$Client,
        [string]$Operation,
        [hashtable]$Values
    )

    switch ($Operation) {
        "open-session" { return [pscustomobject]@{ sessionId = "dry-run-session-$Client"; reply = [pscustomobject]@{ sessionId = "dry-run-session-$Client" } } }
        "register" { return [pscustomobject]@{ serverHandle = 1; register = [pscustomobject]@{ serverHandle = 1 }; reply = [pscustomobject]@{ register = [pscustomobject]@{ serverHandle = 1 } } } }
        "add-item" { return [pscustomobject]@{ itemHandle = 1; addItem = [pscustomobject]@{ itemHandle = 1 }; reply = [pscustomobject]@{ addItem = [pscustomobject]@{ itemHandle = 1 } } } }
        "write" { return [pscustomobject]@{ ok = $true; operation = "write"; reply = [pscustomobject]@{} } }
        "subscribe-bulk" {
            $results = @($Values.items -split "," | ForEach-Object -Begin { $index = 1 } -Process {
                [pscustomobject]@{ itemHandle = $index++; tagAddress = $_; wasSuccessful = $true }
            })
            return [pscustomobject]@{ subscribeBulk = [pscustomobject]@{ results = $results }; results = $results }
        }
        "unsubscribe-bulk" {
            $results = @($Values.itemHandles -split "," | ForEach-Object {
                [pscustomobject]@{ itemHandle = [int]$_; wasSuccessful = $true }
            })
            return [pscustomobject]@{ unsubscribeBulk = [pscustomobject]@{ results = $results }; results = $results }
        }
        "stream-events" {
            # Echo the requested write value back so the write round-trip
            # assertion passes under -DryRun. The reply is shaped per client:
            # Go and Java emit one event object per line (Read-JsonObject
            # collapses NDJSON to a bare array), the others aggregate the
            # events under an `events` property.
            $itemHandle = if ($Values.ContainsKey("echoItemHandle")) { [int]$Values.echoItemHandle } else { 1 }
            $echoValue = if ($Values.ContainsKey("echoValue")) { $Values.echoValue } else { 1 }
            $event = [pscustomobject]@{
                workerSequence = 1
                itemHandle = $itemHandle
                value = [pscustomobject]@{ int32Value = $echoValue }
            }
            switch ($Client) {
                "go" { return ,@($event) }
                "java" { return ,@($event) }
                "rust" { return [pscustomobject]@{ eventCount = 1; events = @($event) } }
                default { return [pscustomobject]@{ events = @($event) } }
            }
        }
        default { return [pscustomobject]@{ ok = $true; reply = [pscustomobject]@{} } }
    }
}

function Invoke-ClientOperation {
    param(
        [string]$Client,
        [string]$Operation,
        [hashtable]$Values = @{},
        [string]$ApiKeyEnvName = $ApiKeyEnv
    )

    $command = Get-ClientCommand -Client $Client -Operation $Operation -Values $Values -ApiKeyEnvName $ApiKeyEnvName
    $result = Invoke-NativeCommand `
        -FilePath $command.file `
        -Arguments $command.args `
        -WorkingDirectory $command.cwd `
        -Environment $command.env
    if ($DryRun) {
        return Get-DryRunReply -Client $Client -Operation $Operation -Values $Values
    }
    return Read-JsonObject -Text $result.stdout
}

# Runs a client operation that is expected to fail, returning the raw process
# result (exit code + stderr) without throwing. Under -DryRun a synthetic
# failure is returned so the parity and auth phases can be exercised offline.
function Invoke-ClientOperationExpectingFailure {
    param(
        [string]$Client,
        [string]$Operation,
        [hashtable]$Values = @{},
        [string]$ApiKeyEnvName = $ApiKeyEnv
    )

    if ($DryRun) {
        $command = Get-ClientCommand -Client $Client -Operation $Operation -Values $Values -ApiKeyEnvName $ApiKeyEnvName
        Write-Host "[dry-run] $(Join-CommandLine -FilePath $command.file -Arguments $command.args)"
        return [pscustomobject]@{ exitCode = 1; stdout = ""; stderr = "[dry-run] synthetic expected failure" }
    }

    $command = Get-ClientCommand -Client $Client -Operation $Operation -Values $Values -ApiKeyEnvName $ApiKeyEnvName
    return Invoke-NativeCommand `
        -FilePath $command.file `
        -Arguments $command.args `
        -WorkingDirectory $command.cwd `
        -Environment $command.env `
        -AllowFailure
}

# Runs the full e2e flow for a single language client and returns the result
# record. Discovered tags are passed in so the (slow) SQL discovery runs once.
function Invoke-ClientFlow {
    param(
        [string]$Client,
        [object[]]$Tags
    )

    Write-Host "Running $Client client e2e flow against $($Tags.Count) discovered tags."
    $sessionId = $null
    $serverHandle = $null
    $clientResult = [ordered]@{
        language = $Client
        sessionId = $null
        serverHandle = $null
        bulk = $null
        addedItems = @()
        eventCount = 0
        write = $null
        parity = @()
        auth = @()
        closed = $false
        error = $null
    }

    try {
        $openJson = Invoke-ClientOperation -Client $Client -Operation "open-session"
        $sessionId = Get-OpenSessionId -Client $Client -Json $openJson
        if ([string]::IsNullOrWhiteSpace($sessionId)) {
            throw "The $Client open-session command did not return a session id."
        }
        $clientResult.sessionId = $sessionId

        $registerJson = Invoke-ClientOperation -Client $Client -Operation "register" -Values @{
            sessionId = $sessionId
        }
        $serverHandle = Get-ServerHandle -Client $Client -Json $registerJson
        $clientResult.serverHandle = $serverHandle

        if (-not $SkipBulk) {
            $bulkTags = @($Tags | Select-Object -First ([Math]::Min($BulkTagCount, $Tags.Count)))
            $bulkItems = ($bulkTags | ForEach-Object { $_.fullTagReference }) -join ","
            $subscribeBulkJson = Invoke-ClientOperation -Client $Client -Operation "subscribe-bulk" -Values @{
                sessionId = $sessionId
                serverHandle = $serverHandle
                items = $bulkItems
            }
            $subscribeResults = @(Get-BulkResults -Client $Client -Operation "subscribe-bulk" -Json $subscribeBulkJson)
            Assert-BulkResults -Client $Client -Operation "subscribe-bulk" -Results $subscribeResults -ExpectedCount $bulkTags.Count
            $bulkItemHandles = @(Get-BulkItemHandles -Results $subscribeResults)
            if ($bulkItemHandles.Count -ne $bulkTags.Count) {
                throw "$Client subscribe-bulk returned $($bulkItemHandles.Count) usable item handle(s); expected $($bulkTags.Count)."
            }

            $unsubscribeBulkJson = Invoke-ClientOperation -Client $Client -Operation "unsubscribe-bulk" -Values @{
                sessionId = $sessionId
                serverHandle = $serverHandle
                itemHandles = $bulkItemHandles -join ","
            }
            $unsubscribeResults = @(Get-BulkResults -Client $Client -Operation "unsubscribe-bulk" -Json $unsubscribeBulkJson)
            Assert-BulkResults -Client $Client -Operation "unsubscribe-bulk" -Results $unsubscribeResults -ExpectedCount $bulkItemHandles.Count

            $clientResult.bulk = [ordered]@{
                tagCount = $bulkTags.Count
                subscribedCount = $subscribeResults.Count
                unsubscribedCount = $unsubscribeResults.Count
                itemHandles = $bulkItemHandles
            }
        }

        foreach ($tag in $Tags) {
            $addJson = Invoke-ClientOperation -Client $Client -Operation "add-item" -Values @{
                sessionId = $sessionId
                serverHandle = $serverHandle
                item = $tag.fullTagReference
            }
            $itemHandle = Get-ItemHandle -Client $Client -Json $addJson
            Invoke-ClientOperation -Client $Client -Operation "advise" -Values @{
                sessionId = $sessionId
                serverHandle = $serverHandle
                itemHandle = $itemHandle
            } | Out-Null

            $clientResult.addedItems += [ordered]@{
                tagName = $tag.tagName
                attributeName = $tag.attributeName
                fullTagReference = $tag.fullTagReference
                itemHandle = $itemHandle
                protectedWriteRequired = $tag.attributeName -eq "ProtectedValue"
            }
        }

        # --- Write round-trip + value assertion ---------------------------
        # Write a per-client sentinel value to a configured writable
        # attribute, then assert it is echoed back through the event stream.
        $writeTarget = $null
        if (-not $SkipWrite) {
            $writeTarget = @($clientResult.addedItems | Where-Object {
                $_.attributeName -eq $WriteAttribute
            }) | Select-Object -First 1
        }

        $doWrite = $null -ne $writeTarget
        $sentinelValue = $null
        if ($doWrite) {
            $sentinelValue = "$($WriteValueBase + $script:clientFlowIndex)"
            Invoke-ClientOperation -Client $Client -Operation "write" -Values @{
                sessionId = $sessionId
                serverHandle = $serverHandle
                itemHandle = $writeTarget.itemHandle
                valueType = $WriteType
                value = $sentinelValue
            } | Out-Null
        } elseif (-not $SkipWrite) {
            Write-Warning "$Client write phase skipped: no discovered tag has attribute '$WriteAttribute'."
        }

        # --- Event streaming (also serves the write echo assertion) -------
        $captureEvents = (-not $SkipStream) -or $doWrite
        if ($captureEvents) {
            $streamValues = @{ sessionId = $sessionId }
            if ($doWrite) {
                $streamValues.maxEvents = $WriteEchoMaxEvents
                $streamValues.echoItemHandle = $writeTarget.itemHandle
                $streamValues.echoValue = $sentinelValue
            }
            $streamJson = Invoke-ClientOperation -Client $Client -Operation "stream-events" -Values $streamValues
            $events = @(Get-StreamEvents -Client $Client -Json $streamJson)
            $clientResult.eventCount = Get-StreamEventCount -Client $Client -Json $streamJson

            if (-not $SkipStream -and $clientResult.eventCount -lt 1) {
                throw "The $Client stream-events command returned no events."
            }

            if ($doWrite) {
                $echoEvent = $events | Where-Object {
                    (Get-EventItemHandle -Event $_) -eq $writeTarget.itemHandle -and
                    (Test-ValueEquals -Expected $sentinelValue -Observed (Get-EventScalar -Event $_))
                } | Select-Object -First 1

                if ($null -eq $echoEvent) {
                    throw ("$Client write round-trip failed: wrote $WriteType=$sentinelValue to " +
                        "'$($writeTarget.fullTagReference)' (item handle $($writeTarget.itemHandle)) " +
                        "but no matching value was observed in $($events.Count) streamed event(s). " +
                        "Increase -WriteEchoMaxEvents, point -WriteAttribute at a writable attribute, or pass -SkipWrite.")
                }

                $clientResult.write = [ordered]@{
                    attributeName = $WriteAttribute
                    fullTagReference = $writeTarget.fullTagReference
                    itemHandle = $writeTarget.itemHandle
                    valueType = $WriteType
                    value = $sentinelValue
                    echoObserved = $true
                    echoWorkerSequence = (Get-PropertyValue -Object $echoEvent -Names @("workerSequence", "worker_sequence"))
                }
            }
        }

        # --- Error-path (parity) checks -----------------------------------
        # MXAccess parity: an invalid item handle and an unknown session must
        # both be rejected rather than silently succeeding.
        if (-not $SkipParity) {
            $parityChecks = @(
                [ordered]@{
                    check = "invalid-item-handle"
                    operation = "advise"
                    values = @{ sessionId = $sessionId; serverHandle = $serverHandle; itemHandle = 2147483647 }
                },
                [ordered]@{
                    check = "unknown-session"
                    operation = "register"
                    values = @{ sessionId = [guid]::NewGuid().ToString() }
                }
            )

            foreach ($parityCheck in $parityChecks) {
                $parityResult = Invoke-ClientOperationExpectingFailure `
                    -Client $Client -Operation $parityCheck.operation -Values $parityCheck.values
                $passed = $parityResult.exitCode -ne 0
                $clientResult.parity += [ordered]@{
                    check = $parityCheck.check
                    operation = $parityCheck.operation
                    exitCode = $parityResult.exitCode
                    passed = $passed
                }
                if (-not $passed) {
                    throw "$Client parity check '$($parityCheck.check)' expected $($parityCheck.operation) to fail but it exited 0."
                }
            }
        }

        # --- API-key auth rejection ---------------------------------------
        # Runs after a working session is established, so a non-zero exit is
        # an auth rejection rather than the gateway being unreachable.
        if (-not $SkipAuth) {
            $authChecks = @(
                [ordered]@{ check = "missing-api-key"; apiKeyEnv = $script:missingKeyEnvName }
            )
            if (-not [string]::IsNullOrWhiteSpace($RejectScopeApiKeyEnv)) {
                $authChecks += [ordered]@{ check = "insufficient-scope"; apiKeyEnv = $RejectScopeApiKeyEnv }
            }

            foreach ($authCheck in $authChecks) {
                $authResult = Invoke-ClientOperationExpectingFailure `
                    -Client $Client -Operation "open-session" -ApiKeyEnvName $authCheck.apiKeyEnv
                $passed = $authResult.exitCode -ne 0
                $clientResult.auth += [ordered]@{
                    check = $authCheck.check
                    exitCode = $authResult.exitCode
                    passed = $passed
                }
                if (-not $passed) {
                    throw "$Client auth check '$($authCheck.check)' expected open-session to be rejected but it exited 0."
                }
            }
        }
    } catch {
        $clientResult.error = $_.Exception.Message
        Write-Warning "$Client e2e flow failed: $($clientResult.error)"
    } finally {
        if (-not [string]::IsNullOrWhiteSpace($sessionId)) {
            try {
                Invoke-ClientOperation -Client $Client -Operation "close-session" -Values @{
                    sessionId = $sessionId
                } | Out-Null
                $clientResult.closed = $true
            } catch {
                $clientResult.error = "$($clientResult.error) close-session failed: $($_.Exception.Message)"
            }
        }
    }

    return $clientResult
}

# Forwards every run parameter to a single-client child invocation used by
# -Parallel. -Parallel itself is intentionally omitted so the child runs the
# serial path.
function Get-ChildArgumentList {
    param(
        [string]$Client,
        [string]$ChildReportPath
    )

    $childArgs = @(
        "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $PSCommandPath,
        "-Clients", $Client,
        "-MachineStart", "$MachineStart",
        "-MachineEnd", "$MachineEnd",
        "-Attributes", ($Attributes -join ","),
        "-Endpoint", $Endpoint,
        "-ApiKeyEnv", $ApiKeyEnv,
        "-SqlServer", $SqlServer,
        "-Database", $Database,
        "-EventLimit", "$EventLimit",
        "-BulkTagCount", "$BulkTagCount",
        "-WriteAttribute", $WriteAttribute,
        "-WriteType", $WriteType,
        "-WriteValueBase", "$WriteValueBase",
        "-WriteEchoMaxEvents", "$WriteEchoMaxEvents",
        "-ReportPath", $ChildReportPath,
        "-EmitReport"
    )
    if (-not [string]::IsNullOrWhiteSpace($RejectScopeApiKeyEnv)) {
        $childArgs += @("-RejectScopeApiKeyEnv", $RejectScopeApiKeyEnv)
    }
    if ($SkipStream) { $childArgs += "-SkipStream" }
    if ($SkipBulk) { $childArgs += "-SkipBulk" }
    if ($SkipWrite) { $childArgs += "-SkipWrite" }
    if ($SkipParity) { $childArgs += "-SkipParity" }
    if ($SkipAuth) { $childArgs += "-SkipAuth" }
    if ($DryRun) { $childArgs += "-DryRun" }
    return $childArgs
}

# An env var name that is guaranteed not to be set in this process, used to
# drive the missing-API-key auth rejection.
$script:missingKeyEnvName = "MXGW_E2E_MISSING_KEY_" + ([guid]::NewGuid().ToString("N"))

# --- Parallel mode: fan out one isolated child process per client ----------
if ($Parallel -and $Clients.Count -gt 1) {
    Write-Host "Running $($Clients.Count) client e2e flows in parallel."
    $childRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("mxgw-e2e-" + ([guid]::NewGuid().ToString("N")))
    New-Item -ItemType Directory -Path $childRoot -Force | Out-Null

    $children = @()
    foreach ($client in $Clients) {
        $childReport = Join-Path $childRoot "$client.json"
        $childLog = Join-Path $childRoot "$client.log"
        $childArgs = Get-ChildArgumentList -Client $client -ChildReportPath $childReport
        $process = Start-Process -FilePath "pwsh" -ArgumentList $childArgs `
            -RedirectStandardOutput $childLog -RedirectStandardError "$childLog.err" `
            -NoNewWindow -PassThru
        $children += [pscustomobject]@{
            client = $client
            process = $process
            report = $childReport
            log = $childLog
        }
    }

    foreach ($child in $children) {
        $child.process.WaitForExit()
    }

    $mergedClients = @()
    $discoveredTags = @()
    $hadFailure = $false
    foreach ($child in $children) {
        foreach ($logPath in @($child.log, "$($child.log).err")) {
            if ((Test-Path $logPath) -and -not [string]::IsNullOrWhiteSpace((Get-Content -Raw -Path $logPath))) {
                Write-Host "----- $($child.client) -----"
                Get-Content -Path $logPath | ForEach-Object { Write-Host $_ }
            }
        }

        if ($child.process.ExitCode -ne 0) {
            $hadFailure = $true
        }

        if (Test-Path $child.report) {
            $childRun = Get-Content -Raw -Path $child.report | ConvertFrom-Json
            $mergedClients += @($childRun.clients)
            if ($discoveredTags.Count -eq 0) {
                $discoveredTags = @($childRun.discoveredTags)
            }
        } else {
            $hadFailure = $true
            $mergedClients += [pscustomobject]@{
                language = $child.client
                error = "Child process exited $($child.process.ExitCode) without writing a report."
            }
        }
    }

    $run = [ordered]@{
        schemaVersion = 1
        endpoint = $Endpoint
        apiKeyEnv = $ApiKeyEnv
        machineStart = $MachineStart
        machineEnd = $MachineEnd
        attributes = $Attributes
        eventLimit = $EventLimit
        bulkTagCount = $BulkTagCount
        skipStream = [bool]$SkipStream
        skipBulk = [bool]$SkipBulk
        skipWrite = [bool]$SkipWrite
        skipParity = [bool]$SkipParity
        skipAuth = [bool]$SkipAuth
        writeAttribute = $WriteAttribute
        parallel = $true
        discoveredTags = $discoveredTags
        clients = $mergedClients
        completedAt = (Get-Date).ToUniversalTime().ToString("O")
        success = -not $hadFailure
    }

    $reportDirectory = Split-Path -Parent $ReportPath
    if (-not [string]::IsNullOrWhiteSpace($reportDirectory)) {
        New-Item -ItemType Directory -Path $reportDirectory -Force | Out-Null
    }
    $run | ConvertTo-Json -Depth 8 | Set-Content -Path $ReportPath -Encoding UTF8
    Write-Host "Wrote merged e2e report to $ReportPath"
    Remove-Item -Path $childRoot -Recurse -Force -ErrorAction SilentlyContinue

    if ($hadFailure) {
        exit 1
    }
    return
}

# --- Serial mode -----------------------------------------------------------
$discoveryJson = & $discoveryScript `
    -MachineStart $MachineStart `
    -MachineEnd $MachineEnd `
    -Attributes $Attributes `
    -SqlServer $SqlServer `
    -Database $Database `
    -Json
$convertedTags = $discoveryJson | ConvertFrom-Json
$tags = @($convertedTags)
if ($tags.Count -eq 1 -and $tags[0] -is [System.Array]) {
    $tags = @($tags[0])
}
$expectedTagCount = (($MachineEnd - $MachineStart + 1) * $Attributes.Count)

if ($tags.Count -ne $expectedTagCount) {
    $found = $tags | Group-Object -Property tagName | ForEach-Object {
        "$($_.Name)=$($_.Count)"
    }
    throw "Expected $expectedTagCount discovered test tags, found $($tags.Count): $($found -join ', ')"
}

$run = [ordered]@{
    schemaVersion = 1
    endpoint = $Endpoint
    apiKeyEnv = $ApiKeyEnv
    machineStart = $MachineStart
    machineEnd = $MachineEnd
    attributes = $Attributes
    eventLimit = $EventLimit
    bulkTagCount = $BulkTagCount
    skipStream = [bool]$SkipStream
    skipBulk = [bool]$SkipBulk
    skipWrite = [bool]$SkipWrite
    skipParity = [bool]$SkipParity
    skipAuth = [bool]$SkipAuth
    writeAttribute = $WriteAttribute
    parallel = $false
    startedAt = (Get-Date).ToUniversalTime().ToString("O")
    discoveredTags = $tags
    clients = @()
}

$hadFailure = $false
$script:clientFlowIndex = 0

foreach ($client in $Clients) {
    $clientResult = Invoke-ClientFlow -Client $client -Tags $tags
    if (-not [string]::IsNullOrWhiteSpace($clientResult.error)) {
        $hadFailure = $true
    }
    $run.clients += $clientResult
    $script:clientFlowIndex++
}

$run.completedAt = (Get-Date).ToUniversalTime().ToString("O")
$run.success = -not $hadFailure

if (-not $DryRun -or $EmitReport) {
    $reportDirectory = Split-Path -Parent $ReportPath
    if (-not [string]::IsNullOrWhiteSpace($reportDirectory)) {
        New-Item -ItemType Directory -Path $reportDirectory -Force | Out-Null
    }
    $run | ConvertTo-Json -Depth 8 | Set-Content -Path $ReportPath -Encoding UTF8
    Write-Host "Wrote e2e report to $ReportPath"
}

if ($hadFailure) {
    exit 1
}