dohertj2/mxaccessgw
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Resolve Client.Go-022..027: bulk flags, bench cancel, batch loop

Browse Source 
Client.Go-022  Re-applied Client.Go-015 shape — runWriteBulkVariant drops
               the unused secured param and gates -current-user-id /
               -verifier-user-id / -user-id behind the secured-only
               variants.
Client.Go-023  Re-applied Client.Go-018 shape — bench warm-up and steady-
               state loops respect ctx.Err().
Client.Go-024  Added SDK-level tests for WriteBulk / Write2Bulk /
               WriteSecuredBulk / WriteSecured2Bulk / ReadBulk and
               StreamAlarms via the existing bufconn fake gateway pattern.
Client.Go-025  Five bulk SDK methods short-circuit on empty input without
               an RPC round-trip and document the behavior.
Client.Go-026  runBatch widens scanner.Buffer to 16 MiB and emits an
               error-with-sentinel if a longer line still arrives, rather
               than aborting the session silently.
Client.Go-027  runBatch treats blank lines as skip-and-continue; only EOF
               ends the session.

All resolved at 2026-05-24; gofmt + go vet + go build + go test ./... all
green.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
2026-05-24 08:49:58 -04:00
parent 712cb06442
commit 82996aa8e6
6 changed files with 588 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clients/go/cmd/mxgw-go/main.go
+59 -20
View File
@@ -396,25 +396,31 @@ func runReadBulk(ctx context.Context, args []string, stdout, stderr io.Writer) e
}
func runWriteBulk(ctx context.Context, args []string, stdout, stderr io.Writer) error {
return runWriteBulkVariant(ctx, args, stdout, stderr, "write-bulk", false, false)
return runWriteBulkVariant(ctx, args, stdout, stderr, "write-bulk", false)
}
func runWrite2Bulk(ctx context.Context, args []string, stdout, stderr io.Writer) error {
return runWriteBulkVariant(ctx, args, stdout, stderr, "write2-bulk", true, false)
return runWriteBulkVariant(ctx, args, stdout, stderr, "write2-bulk", true)
}
func runWriteSecuredBulk(ctx context.Context, args []string, stdout, stderr io.Writer) error {
return runWriteBulkVariant(ctx, args, stdout, stderr, "write-secured-bulk", false, true)
return runWriteBulkVariant(ctx, args, stdout, stderr, "write-secured-bulk", false)
}
func runWriteSecured2Bulk(ctx context.Context, args []string, stdout, stderr io.Writer) error {
return runWriteBulkVariant(ctx, args, stdout, stderr, "write-secured2-bulk", true, true)
return runWriteBulkVariant(ctx, args, stdout, stderr, "write-secured2-bulk", true)
}
// runWriteBulkVariant shares the flag-parsing + entry-build skeleton across
// the four bulk-write families. withTimestamp adds a --timestamp-value flag;
// secured switches from --user-id to --current-user-id / --verifier-user-id.
func runWriteBulkVariant(ctx context.Context, args []string, stdout, stderr io.Writer, command string, withTimestamp bool, secured bool) error {
// the four bulk-write families. The variant is derived from command alone;
// withTimestamp adds a --timestamp-value flag. To keep wrong-variant flags
// from silently no-op'ing, secured-only flags (-current-user-id /
// -verifier-user-id) are only registered for the secured variants, and
// -user-id only for the non-secured Write/Write2 variants — a wrong-variant
// flag then surfaces as a clean "flag provided but not defined" error.
func runWriteBulkVariant(ctx context.Context, args []string, stdout, stderr io.Writer, command string, withTimestamp bool) error {
secured := command == "write-secured-bulk" || command == "write-secured2-bulk"
flags := flag.NewFlagSet(command, flag.ContinueOnError)
flags.SetOutput(stderr)
common := bindCommonFlags(flags)
@@ -424,9 +430,17 @@ func runWriteBulkVariant(ctx context.Context, args []string, stdout, stderr io.W
itemHandles := flags.String("item-handles", "", "comma-separated item handles")
valueType := flags.String("type", "string", "value type: bool, int32, int64, float, double, string")
values := flags.String("values", "", "comma-separated values (one per item handle)")
userID := flags.Int("user-id", 0, "MXAccess user id (Write/Write2 variants)")
currentUserID := flags.Int("current-user-id", 0, "MXAccess current user id (Secured variants)")
verifierUserID := flags.Int("verifier-user-id", 0, "MXAccess verifier user id (Secured variants)")
var (
userID *int
currentUserID *int
verifierUserID *int
)
if secured {
currentUserID = flags.Int("current-user-id", 0, "MXAccess current user id (Secured variants)")
verifierUserID = flags.Int("verifier-user-id", 0, "MXAccess verifier user id (Secured variants)")
} else {
userID = flags.Int("user-id", 0, "MXAccess user id (Write/Write2 variants)")
}
timestampValue := flags.String("timestamp-value", "", "RFC 3339 timestamp shared across all entries (Write2/WriteSecured2 variants)")
if err := flags.Parse(args); err != nil {
@@ -514,7 +528,6 @@ func runWriteBulkVariant(ctx context.Context, args []string, stdout, stderr io.W
default:
return fmt.Errorf("unsupported bulk write command %q", command)
}
_ = secured // currently only used for routing above; reserved for future per-variant validation
return writeWriteBulkOutput(stdout, *jsonOutput, command, options, results, err)
}
@@ -598,10 +611,12 @@ func runBenchReadBulk(ctx context.Context, args []string, stdout, stderr io.Writ
}()
// Warm-up: drive identical calls so any first-call JIT / connection-pool
// setup is amortised before the measurement window opens.
// setup is amortised before the measurement window opens. The ctx.Err()
// guard short-circuits on Ctrl+C / parent-cancel instead of spinning
// failing ReadBulk calls until the wall-clock deadline elapses.
warmupDeadline := time.Now().Add(time.Duration(*warmupSeconds) * time.Second)
timeout := time.Duration(*timeoutMs) * time.Millisecond
for time.Now().Before(warmupDeadline) {
for time.Now().Before(warmupDeadline) && ctx.Err() == nil {
_, _ = session.ReadBulk(ctx, serverHandle, tags, timeout)
}
@@ -613,7 +628,7 @@ func runBenchReadBulk(ctx context.Context, args []string, stdout, stderr io.Writ
steadyStart := time.Now()
steadyDeadline := steadyStart.Add(time.Duration(*durationSeconds) * time.Second)
for time.Now().Before(steadyDeadline) {
for time.Now().Before(steadyDeadline) && ctx.Err() == nil {
callStart := time.Now()
results, err := session.ReadBulk(ctx, serverHandle, tags, timeout)
elapsed := time.Since(callStart)
@@ -1191,18 +1206,28 @@ const batchEOR = "__MXGW_BATCH_EOR__"
// runBatch reads one command line at a time from in, dispatches each via the
// normal runWithIO routing, and writes a batchEOR sentinel to stdout after
// every result. Errors are serialised as JSON to stdout (not stderr) so the
// harness can parse them without interleaving stderr. The loop never terminates
// on command error; only stdin EOF (or an empty line) ends the session.
// harness can parse them without interleaving stderr. Blank lines are
// skipped; only stdin EOF ends the session.
//
// The scanner buffer is widened to 16 MiB so a single long command line
// (e.g. a bulk-write with several thousand handles) does not trip the
// default 64 KiB bufio.Scanner token-too-long error and abort the session.
// If a line still exceeds the cap, the error is surfaced as a per-command
// error-with-sentinel and the session continues.
func runBatch(ctx context.Context, in io.Reader, stdout, stderr io.Writer) error {
bw := bufio.NewWriter(stdout)
scanner := bufio.NewScanner(in)
for scanner.Scan() {
line := scanner.Text()
if line == "" {
scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
for {
if !scanner.Scan() {
break
}
line := scanner.Text()
args := strings.Fields(line)
if len(args) == 0 {
// Skip blank / whitespace-only lines; do NOT terminate. The
// session ends only on stdin EOF so a stray blank line in a
// PowerShell here-string does not silently drop later commands.
continue
}
if err := runWithIO(ctx, args, bw, stderr); err != nil {
@@ -1217,7 +1242,21 @@ func runBatch(ctx context.Context, in io.Reader, stdout, stderr io.Writer) error
_, _ = fmt.Fprintln(bw, batchEOR)
_ = bw.Flush()
}
return scanner.Err()
if err := scanner.Err(); err != nil {
// Emit the scanner failure as a final error-with-sentinel so the
// harness sees the failure framed, then return the error so the
// process exit reflects it. This handles bufio.ErrTooLong for any
// pathological line above the 16 MiB cap.
errPayload := map[string]string{
"error": err.Error(),
"type": "error",
}
_ = writeJSON(bw, errPayload)
_, _ = fmt.Fprintln(bw, batchEOR)
_ = bw.Flush()
return err
}
return nil
}
func dialGalaxyForCommand(ctx context.Context, common *commonOptions) (*mxgateway.GalaxyClient, commonOptions, error) {
clients/go/cmd/mxgw-go/main_test.go
+210
View File
@@ -2,9 +2,15 @@ package main
import (
"bytes"
"context"
"encoding/json"
"net"
"strings"
"testing"
"time"
pb "gitea.dohertylan.com/dohertj2/mxaccessgw/clients/go/internal/generated"
"google.golang.org/grpc"
)
func TestRunVersionJSON(t *testing.T) {
@@ -84,3 +90,207 @@ func TestParseValueBuildsTypedValue(t *testing.T) {
t.Fatalf("int32 value = %d, want 123", got)
}
}
// TestRunWriteBulkVariantGatesSecuredFlags pins the Client.Go-022 fix:
// secured-only flags must be unavailable on non-secured variants, and
// vice-versa, so a wrong-variant flag fails with a clean "flag provided
// but not defined" error instead of silently no-op'ing.
func TestRunWriteBulkVariantGatesSecuredFlags(t *testing.T) {
cases := []struct {
name string
args []string
}{
{
name: "write-bulk-rejects-current-user-id",
args: []string{"write-bulk", "-current-user-id", "5", "-item-handles", "1", "-values", "1"},
},
{
name: "write-bulk-rejects-verifier-user-id",
args: []string{"write-bulk", "-verifier-user-id", "5", "-item-handles", "1", "-values", "1"},
},
{
name: "write2-bulk-rejects-current-user-id",
args: []string{"write2-bulk", "-current-user-id", "5", "-item-handles", "1", "-values", "1"},
},
{
name: "write-secured-bulk-rejects-user-id",
args: []string{"write-secured-bulk", "-user-id", "5", "-item-handles", "1", "-values", "1"},
},
{
name: "write-secured2-bulk-rejects-user-id",
args: []string{"write-secured2-bulk", "-user-id", "5", "-item-handles", "1", "-values", "1"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
err := runWithIO(t.Context(), tc.args, &stdout, &stderr)
if err == nil {
t.Fatalf("runWithIO(%v) returned no error", tc.args)
}
if !strings.Contains(err.Error(), "flag provided but not defined") {
t.Fatalf("runWithIO(%v) error = %v; want 'flag provided but not defined'", tc.args, err)
}
})
}
}
// TestRunBenchReadBulkRespectsContextCancellation pins the Client.Go-023
// fix: the warm-up and steady-state wall-clock loops must honour ctx.Err()
// so an external cancel (Ctrl+C, parent-cancel from a cross-language bench
// driver) short-circuits the bench instead of spinning failing ReadBulk
// calls until the wall-clock deadline elapses.
func TestRunBenchReadBulkRespectsContextCancellation(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
server := grpc.NewServer()
fake := &benchFakeGateway{}
pb.RegisterMxAccessGatewayServer(server, fake)
go func() {
_ = server.Serve(listener)
}()
defer server.Stop()
defer listener.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Long warm-up + duration, so if the ctx.Err() guard were missing the
// loops would run for ~10s. With the guard, the cancel below short-
// circuits both loops within ~one ReadBulk iteration.
args := []string{
"bench-read-bulk",
"-endpoint", listener.Addr().String(),
"-plaintext",
"-api-key", "test",
"-warmup-seconds", "5",
"-duration-seconds", "5",
"-bulk-size", "1",
"-timeout-ms", "100",
}
// Cancel after a brief delay — far less than warmup+duration (10s).
go func() {
time.Sleep(150 * time.Millisecond)
cancel()
}()
var stdout, stderr bytes.Buffer
start := time.Now()
err = runWithIO(ctx, args, &stdout, &stderr)
elapsed := time.Since(start)
// With the ctx.Err() guard, the loops exit well before the wall-clock
// deadlines (warmup=5s + duration=5s = 10s). Allow generous slack for
// CI noise but assert clearly less than the un-guarded worst case.
if elapsed > 4*time.Second {
t.Fatalf("bench-read-bulk took %s after ctx cancel; want <4s (ctx.Err() guard missing?). err=%v stderr=%s", elapsed, err, stderr.String())
}
}
// benchFakeGateway is a minimal MxAccessGatewayServer that satisfies the
// bench-read-bulk session-setup sequence (OpenSession + Invoke for Register
// / SubscribeBulk / ReadBulk / UnsubscribeBulk / CloseSession).
type benchFakeGateway struct {
pb.UnimplementedMxAccessGatewayServer
}
func (g *benchFakeGateway) OpenSession(_ context.Context, _ *pb.OpenSessionRequest) (*pb.OpenSessionReply, error) {
return &pb.OpenSessionReply{
SessionId: "bench-session",
ProtocolStatus: &pb.ProtocolStatus{Code: pb.ProtocolStatusCode_PROTOCOL_STATUS_CODE_OK},
}, nil
}
func (g *benchFakeGateway) CloseSession(_ context.Context, req *pb.CloseSessionRequest) (*pb.CloseSessionReply, error) {
return &pb.CloseSessionReply{
SessionId: req.GetSessionId(),
ProtocolStatus: &pb.ProtocolStatus{Code: pb.ProtocolStatusCode_PROTOCOL_STATUS_CODE_OK},
}, nil
}
func (g *benchFakeGateway) Invoke(_ context.Context, req *pb.MxCommandRequest) (*pb.MxCommandReply, error) {
kind := req.GetCommand().GetKind()
reply := &pb.MxCommandReply{
SessionId: req.GetSessionId(),
Kind: kind,
ProtocolStatus: &pb.ProtocolStatus{Code: pb.ProtocolStatusCode_PROTOCOL_STATUS_CODE_OK},
}
switch kind {
case pb.MxCommandKind_MX_COMMAND_KIND_REGISTER:
reply.Payload = &pb.MxCommandReply_Register{Register: &pb.RegisterReply{ServerHandle: 1}}
case pb.MxCommandKind_MX_COMMAND_KIND_SUBSCRIBE_BULK:
reply.Payload = &pb.MxCommandReply_SubscribeBulk{SubscribeBulk: &pb.BulkSubscribeReply{
Results: []*pb.SubscribeResult{{ServerHandle: 1, ItemHandle: 1, WasSuccessful: true}},
}}
case pb.MxCommandKind_MX_COMMAND_KIND_READ_BULK:
reply.Payload = &pb.MxCommandReply_ReadBulk{ReadBulk: &pb.BulkReadReply{
Results: []*pb.BulkReadResult{{ItemHandle: 1, WasSuccessful: true, WasCached: true}},
}}
case pb.MxCommandKind_MX_COMMAND_KIND_UNSUBSCRIBE_BULK:
reply.Payload = &pb.MxCommandReply_UnsubscribeBulk{UnsubscribeBulk: &pb.BulkSubscribeReply{}}
}
return reply, nil
}
// TestRunBenchReadBulkRejectsNonPositiveBulkSize pins the Client.Go-023-adjacent
// positivity checks so they cannot drift while resolving the cancellation finding.
func TestRunBenchReadBulkRejectsNonPositiveBulkSize(t *testing.T) {
var stdout, stderr bytes.Buffer
err := runWithIO(t.Context(), []string{"bench-read-bulk", "-bulk-size", "0"}, &stdout, &stderr)
if err == nil || !strings.Contains(err.Error(), "bulk-size must be positive") {
t.Fatalf("bench-read-bulk -bulk-size 0 error = %v", err)
}
}
// TestRunBatchSkipsBlankLinesAndContinuesUntilEOF pins the Client.Go-027 fix:
// a blank line in the middle of a batch session must NOT terminate the loop —
// only stdin EOF ends the session.
func TestRunBatchSkipsBlankLinesAndContinuesUntilEOF(t *testing.T) {
var stdout, stderr bytes.Buffer
// version -> blank -> version (a stray blank line in the middle of a
// programmatic session).
in := strings.NewReader("version --json\n\nversion --json\n")
if err := runBatch(t.Context(), in, &stdout, &stderr); err != nil {
t.Fatalf("runBatch() error = %v; stderr = %s", err, stderr.String())
}
out := stdout.String()
// Both version commands must have produced a result before the EOR sentinel.
if count := strings.Count(out, batchEOR); count != 2 {
t.Fatalf("EOR sentinel count = %d, want 2 (one per command, blank line skipped); out = %q", count, out)
}
}
// TestRunBatchHandlesLongCommandLine pins the Client.Go-026 fix: a command
// line longer than the default bufio.Scanner token size (64 KiB) must not
// abort the batch session.
func TestRunBatchHandlesLongCommandLine(t *testing.T) {
var stdout, stderr bytes.Buffer
// Build a single command line larger than 64 KiB. The command itself is
// invalid (no real session) but runBatch must still emit an EOR sentinel
// and continue to the next command rather than dropping the line on the
// floor with a bufio.ErrTooLong from the outer return.
huge := strings.Repeat("tag-with-a-reasonably-long-name-and-suffix,", 2000) + "trailing"
line := "subscribe-bulk -session-id none -items " + huge
if len(line) <= 64*1024 {
t.Fatalf("test setup error: long line length = %d, want > 64KiB", len(line))
}
in := strings.NewReader(line + "\nversion --json\n")
if err := runBatch(t.Context(), in, &stdout, &stderr); err != nil {
t.Fatalf("runBatch() error = %v; stderr = %s", err, stderr.String())
}
out := stdout.String()
// Both commands must produce an EOR sentinel — the long line should be a
// per-command error (still emitted with EOR), then the version command
// should run normally.
if count := strings.Count(out, batchEOR); count != 2 {
t.Fatalf("EOR sentinel count = %d, want 2 (one per command, even when first is too long); out length = %d", count, len(out))
}
}