feat(sitecallaudit): PullSiteCalls reconciliation plumbing (store read + RPC + site handler + central client)

Site Call Audit (#22): build the documented periodic reconciliation PULL
self-heal path for the eventually-consistent central SiteCalls mirror, as a
dedicated PullSiteCalls gRPC RPC kept separate from the audit pull. This is the
pull PLUMBING only; the central reconciliation tick is a separate follow-up.

- IOperationTrackingStore.ReadChangedSinceAsync(sinceUtc, batchSize): inclusive
  UpdatedAtUtc cursor, oldest-first, batch-capped; SQLite impl projects tracking
  rows onto SiteCallOperational (Kind->Channel, TargetSummary->Target, SourceSite
  left empty - the store has no site-id column).
- sitestream.proto: rpc PullSiteCalls + PullSiteCallsRequest/Response, mirroring
  PullAuditEvents; regenerated checked-in SiteStreamGrpc/*.cs.
- SiteCallDtoMapper.ToDto(SiteCallOperational): inverse of FromDto for the handler.
- SiteStreamGrpcServer.PullSiteCalls handler + SetOperationTrackingStore seam;
  Host wires the seam alongside SetSiteAuditQueue (site roles only).
- Central IPullSiteCallsClient + GrpcPullSiteCallsClient (home: AuditLog/Central to
  reuse ISiteEnumerator; SiteCallAudit does not reference AuditLog). Re-stamps
  SourceSite from the dialed siteId; no-throw on tolerable transport faults;
  SpecifyKind (not ToUniversalTime) cursor handling. Central-only DI registration.

Tests: ReadChangedSinceAsync (4), PullSiteCalls handler (6), GrpcPullSiteCallsClient
(8). Full solution build 0 warnings/0 errors (TreatWarningsAsErrors).
This commit is contained in:
Joseph Doherty
2026-06-15 10:39:06 -04:00
parent c092e89fd1
commit 963e3427da
15 changed files with 1751 additions and 23 deletions
@@ -0,0 +1,215 @@
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
using ProtoPullRequest = ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullSiteCallsRequest;
using ProtoPullResponse = ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullSiteCallsResponse;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central;
/// <summary>
/// Tests for <see cref="GrpcPullSiteCallsClient"/> — the production
/// <see cref="IPullSiteCallsClient"/> that dials a site over gRPC and issues the
/// <c>PullSiteCalls</c> unary RPC for the Site Call Audit (#22) reconciliation
/// loop. The real <c>GrpcChannel</c> is replaced by an injected
/// <see cref="GrpcPullSiteCallsClient.IPullSiteCallsInvoker"/> seam so the
/// client's mapping / ordering / SourceSite-restamp / fault-swallowing behaviour
/// can be asserted without standing up a Kestrel HTTP/2 endpoint. Mirrors
/// <see cref="GrpcPullAuditEventsClientTests"/>.
/// </summary>
public class GrpcPullSiteCallsClientTests
{
private static readonly DateTime BaseTime =
new(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
private sealed class StaticEnumerator : ISiteEnumerator
{
private readonly IReadOnlyList<SiteEntry> _sites;
public StaticEnumerator(params SiteEntry[] sites) => _sites = sites;
public Task<IReadOnlyList<SiteEntry>> EnumerateAsync(CancellationToken ct = default) =>
Task.FromResult(_sites);
}
private sealed class FakeInvoker : GrpcPullSiteCallsClient.IPullSiteCallsInvoker
{
public string? Endpoint { get; private set; }
public ProtoPullRequest? Request { get; private set; }
public int CallCount { get; private set; }
private readonly ProtoPullResponse? _response;
private readonly Exception? _throw;
private FakeInvoker(ProtoPullResponse? response, Exception? toThrow)
{
_response = response;
_throw = toThrow;
}
public static FakeInvoker Returning(ProtoPullResponse response) => new(response, null);
public static FakeInvoker Throwing(Exception ex) => new(null, ex);
public Task<ProtoPullResponse> InvokeAsync(
string endpoint, ProtoPullRequest request, CancellationToken ct)
{
CallCount++;
Endpoint = endpoint;
Request = request;
if (_throw is not null)
{
throw _throw;
}
return Task.FromResult(_response!);
}
}
// The site leaves SourceSite empty (it is not a tracking-store column); the
// client re-stamps it from the dialed siteId. Mint DTOs with empty SourceSite
// to prove that re-stamp.
private static SiteCallOperationalDto Dto(Guid id, DateTime updatedAtUtc) =>
new()
{
TrackedOperationId = id.ToString(),
Channel = "ApiOutbound",
Target = "ERP.GetOrder",
SourceSite = string.Empty,
SourceNode = "node-a",
Status = "Attempted",
RetryCount = 1,
LastError = string.Empty,
CreatedAtUtc = Timestamp.FromDateTime(BaseTime),
UpdatedAtUtc = Timestamp.FromDateTime(updatedAtUtc),
};
[Fact]
public async Task PullAsync_dials_resolved_endpoint_maps_oldest_first_and_restamps_source_site()
{
var older = Guid.NewGuid();
var newer = Guid.NewGuid();
// Wire delivered newest-first on purpose to prove the client sorts.
var proto = new ProtoPullResponse { MoreAvailable = true };
proto.Operationals.Add(Dto(newer, BaseTime.AddMinutes(5)));
proto.Operationals.Add(Dto(older, BaseTime));
var invoker = FakeInvoker.Returning(proto);
var sut = new GrpcPullSiteCallsClient(
new StaticEnumerator(new SiteEntry("site-a", "http://site-a:8083")),
invoker,
NullLogger<GrpcPullSiteCallsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
// Endpoint resolution + request shaping.
Assert.Equal("http://site-a:8083", invoker.Endpoint);
Assert.NotNull(invoker.Request);
Assert.Equal(256, invoker.Request!.BatchSize);
Assert.Equal(BaseTime, invoker.Request.SinceUtc.ToDateTime());
// Mapping + ordering + MoreAvailable surface.
Assert.True(result.MoreAvailable);
Assert.Equal(2, result.SiteCalls.Count);
Assert.Equal(older, result.SiteCalls[0].TrackedOperationId.Value);
Assert.Equal(newer, result.SiteCalls[1].TrackedOperationId.Value);
// SourceSite re-stamped from the dialed siteId (DTO carried empty).
Assert.Equal("site-a", result.SiteCalls[0].SourceSite);
Assert.Equal("site-a", result.SiteCalls[1].SourceSite);
// Round-tripped fields survive FromDto.
Assert.Equal("ApiOutbound", result.SiteCalls[0].Channel);
Assert.Equal("node-a", result.SiteCalls[0].SourceNode);
Assert.Equal(1, result.SiteCalls[0].RetryCount);
}
[Fact]
public async Task PullAsync_returns_empty_when_site_endpoint_is_unknown()
{
var invoker = FakeInvoker.Returning(new ProtoPullResponse());
var sut = new GrpcPullSiteCallsClient(
new StaticEnumerator(), // no sites registered
invoker,
NullLogger<GrpcPullSiteCallsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
Assert.Empty(result.SiteCalls);
Assert.False(result.MoreAvailable);
Assert.Equal(0, invoker.CallCount); // never dialled — nothing to dial
}
[Theory]
[InlineData(StatusCode.Unavailable)]
[InlineData(StatusCode.DeadlineExceeded)]
[InlineData(StatusCode.Cancelled)]
public async Task PullAsync_swallows_tolerable_transport_faults_to_empty_response(StatusCode code)
{
var invoker = FakeInvoker.Throwing(new RpcException(new Status(code, "transport fault")));
var sut = new GrpcPullSiteCallsClient(
new StaticEnumerator(new SiteEntry("site-a", "http://site-a:8083")),
invoker,
NullLogger<GrpcPullSiteCallsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
Assert.Empty(result.SiteCalls);
Assert.False(result.MoreAvailable);
}
[Fact]
public async Task PullAsync_swallows_connection_layer_faults_to_empty_response()
{
var invoker = FakeInvoker.Throwing(new HttpRequestException("connection refused"));
var sut = new GrpcPullSiteCallsClient(
new StaticEnumerator(new SiteEntry("site-a", "http://site-a:8083")),
invoker,
NullLogger<GrpcPullSiteCallsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
Assert.Empty(result.SiteCalls);
Assert.False(result.MoreAvailable);
}
[Fact]
public async Task PullAsync_swallows_unexpected_faults_to_empty_response()
{
var invoker = FakeInvoker.Throwing(new InvalidOperationException("boom"));
var sut = new GrpcPullSiteCallsClient(
new StaticEnumerator(new SiteEntry("site-a", "http://site-a:8083")),
invoker,
NullLogger<GrpcPullSiteCallsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
Assert.Empty(result.SiteCalls);
Assert.False(result.MoreAvailable);
}
[Fact]
public async Task PullAsync_with_minvalue_unspecified_cursor_does_not_throw_and_dials()
{
// The reconciliation cursor starts at DateTime.MinValue with
// Kind=Unspecified. EnsureUtc must treat it AS UTC (per the system-wide
// invariant) and NOT call ToUniversalTime() — on a host with a positive
// UTC offset that underflows and Timestamp.FromDateTime throws, crashing
// the FIRST pull for every site.
var minUnspecified = default(DateTime);
Assert.Equal(DateTimeKind.Unspecified, minUnspecified.Kind);
var invoker = FakeInvoker.Returning(new ProtoPullResponse());
var sut = new GrpcPullSiteCallsClient(
new StaticEnumerator(new SiteEntry("site-a", "http://site-a:8083")),
invoker,
NullLogger<GrpcPullSiteCallsClient>.Instance);
var result = await sut.PullAsync("site-a", minUnspecified, batchSize: 256, CancellationToken.None);
Assert.Equal(1, invoker.CallCount);
Assert.Equal("http://site-a:8083", invoker.Endpoint);
Assert.NotNull(invoker.Request);
Assert.Equal(DateTime.MinValue, invoker.Request!.SinceUtc.ToDateTime());
Assert.Empty(result.SiteCalls);
Assert.False(result.MoreAvailable);
}
}
@@ -0,0 +1,189 @@
using Akka.TestKit.Xunit2;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
/// <summary>
/// Tests for <see cref="SiteStreamGrpcServer.PullSiteCalls"/> (Site Call Audit
/// #22 reconciliation handler). Verifies the request →
/// <see cref="IOperationTrackingStore.ReadChangedSinceAsync"/> → response
/// round-trip through the gRPC handler. The store is an NSubstitute stub so the
/// tests never touch SQLite. Mirrors <see cref="SiteStreamPullAuditEventsTests"/>
/// — but there is no MarkReconciled step (the tracking store is the operational
/// source of truth; the central SiteCalls mirror is upsert-on-newer).
/// </summary>
public class SiteStreamPullSiteCallsTests : TestKit
{
private readonly ISiteStreamSubscriber _subscriber = Substitute.For<ISiteStreamSubscriber>();
private SiteStreamGrpcServer CreateServer() =>
new(_subscriber, NullLogger<SiteStreamGrpcServer>.Instance);
private static ServerCallContext NewContext(CancellationToken ct = default)
{
var context = Substitute.For<ServerCallContext>();
context.CancellationToken.Returns(ct);
return context;
}
private static SiteCallOperational NewOperational() =>
new(
TrackedOperationId: TrackedOperationId.New(),
Channel: "ApiOutbound",
Target: "ERP.GetOrder",
SourceSite: string.Empty,
SourceNode: "node-a",
Status: "Attempted",
RetryCount: 1,
LastError: null,
HttpStatus: 503,
CreatedAtUtc: DateTime.SpecifyKind(new DateTime(2026, 5, 20, 10, 0, 0), DateTimeKind.Utc),
UpdatedAtUtc: DateTime.SpecifyKind(new DateTime(2026, 5, 20, 10, 1, 0), DateTimeKind.Utc),
TerminalAtUtc: null);
[Fact]
public async Task PullSiteCalls_NoStoreWired_ReturnsEmptyResponse()
{
var server = CreateServer();
// Intentionally do NOT call SetOperationTrackingStore — simulates a
// central-only host or a wiring-incomplete startup window.
var request = new PullSiteCallsRequest
{
SinceUtc = Timestamp.FromDateTime(DateTime.UtcNow.AddMinutes(-5)),
BatchSize = 100,
};
var response = await server.PullSiteCalls(request, NewContext());
Assert.Empty(response.Operationals);
Assert.False(response.MoreAvailable);
}
[Fact]
public async Task PullSiteCalls_With5Rows_ReturnsAllFiveDtos()
{
var store = Substitute.For<IOperationTrackingStore>();
var rows = Enumerable.Range(0, 5).Select(_ => NewOperational()).ToList();
store.ReadChangedSinceAsync(Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns((IReadOnlyList<SiteCallOperational>)rows);
var server = CreateServer();
server.SetOperationTrackingStore(store);
var request = new PullSiteCallsRequest
{
SinceUtc = Timestamp.FromDateTime(DateTime.UtcNow.AddHours(-1)),
BatchSize = 100, // larger than returned count so MoreAvailable should be false
};
var response = await server.PullSiteCalls(request, NewContext());
Assert.Equal(5, response.Operationals.Count);
Assert.False(response.MoreAvailable); // 5 < 100
var expectedIds = rows.Select(r => r.TrackedOperationId.ToString()).ToHashSet();
Assert.True(expectedIds.SetEquals(response.Operationals.Select(d => d.TrackedOperationId).ToHashSet()));
}
[Fact]
public async Task PullSiteCalls_PassesSinceUtcThroughVerbatim()
{
var store = Substitute.For<IOperationTrackingStore>();
var capturedSince = DateTime.MinValue;
store.ReadChangedSinceAsync(Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(call =>
{
capturedSince = call.ArgAt<DateTime>(0);
return (IReadOnlyList<SiteCallOperational>)Array.Empty<SiteCallOperational>();
});
var server = CreateServer();
server.SetOperationTrackingStore(store);
var since = DateTime.SpecifyKind(new DateTime(2026, 5, 20, 9, 30, 0), DateTimeKind.Utc);
var request = new PullSiteCallsRequest
{
SinceUtc = Timestamp.FromDateTime(since),
BatchSize = 50,
};
var response = await server.PullSiteCalls(request, NewContext());
Assert.Empty(response.Operationals);
Assert.False(response.MoreAvailable);
Assert.Equal(since, capturedSince);
}
[Fact]
public async Task PullSiteCalls_BatchSize3_Returns3Rows_MoreAvailableTrue()
{
var store = Substitute.For<IOperationTrackingStore>();
var rows = Enumerable.Range(0, 3).Select(_ => NewOperational()).ToList();
store.ReadChangedSinceAsync(Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns((IReadOnlyList<SiteCallOperational>)rows);
var server = CreateServer();
server.SetOperationTrackingStore(store);
var request = new PullSiteCallsRequest
{
SinceUtc = Timestamp.FromDateTime(DateTime.UtcNow.AddHours(-1)),
BatchSize = 3,
};
var response = await server.PullSiteCalls(request, NewContext());
Assert.Equal(3, response.Operationals.Count);
// saturated batch → central needs to know to issue a follow-up pull
Assert.True(response.MoreAvailable);
}
[Fact]
public async Task PullSiteCalls_NonPositiveBatchSize_ThrowsInvalidArgument()
{
var store = Substitute.For<IOperationTrackingStore>();
var server = CreateServer();
server.SetOperationTrackingStore(store);
var request = new PullSiteCallsRequest
{
SinceUtc = Timestamp.FromDateTime(DateTime.UtcNow.AddHours(-1)),
BatchSize = 0,
};
var ex = await Assert.ThrowsAsync<RpcException>(
() => server.PullSiteCalls(request, NewContext()));
Assert.Equal(StatusCode.InvalidArgument, ex.StatusCode);
}
[Fact]
public async Task PullSiteCalls_ReadThrows_ReturnsEmptyResponse()
{
// Best-effort: a read fault must never abort the reconciliation tick.
var store = Substitute.For<IOperationTrackingStore>();
store.ReadChangedSinceAsync(Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
.ThrowsAsync(new InvalidOperationException("SQLite disposed mid-call"));
var server = CreateServer();
server.SetOperationTrackingStore(store);
var request = new PullSiteCallsRequest
{
SinceUtc = Timestamp.FromDateTime(DateTime.UtcNow.AddHours(-1)),
BatchSize = 100,
};
// Must NOT throw — the handler swallows the fault to an empty response.
var response = await server.PullSiteCalls(request, NewContext());
Assert.Empty(response.Operationals);
Assert.False(response.MoreAvailable);
}
}
@@ -439,6 +439,138 @@ public class OperationTrackingStoreTests
Assert.NotNull(await store.GetStatusAsync(cId)); // kept (non-terminal)
}
// ── Site Call Audit #22: ReadChangedSinceAsync (reconciliation pull) ───
[Fact]
public async Task ReadChangedSinceAsync_ReturnsRowsAtOrAfterCursor_OldestFirst()
{
var (store, dataSource) = CreateStore(nameof(ReadChangedSinceAsync_ReturnsRowsAtOrAfterCursor_OldestFirst));
await using var _store = store;
// Three rows with distinct UpdatedAtUtc, written out of chronological
// order to prove the read sorts by UpdatedAtUtc ascending.
var older = TrackedOperationId.New();
var middle = TrackedOperationId.New();
var newer = TrackedOperationId.New();
await store.RecordEnqueueAsync(older, nameof(AuditKind.ApiCallCached), "ERP.A", null, null, "node-a");
await store.RecordEnqueueAsync(middle, nameof(AuditKind.DbWriteCached), "DB.B", null, null, "node-b");
await store.RecordEnqueueAsync(newer, nameof(AuditKind.ApiCallCached), "ERP.C", null, null, null);
// Backdate UpdatedAtUtc so the ordering is deterministic and a cursor
// can be placed cleanly between rows. (Enqueue stamps DateTime.UtcNow;
// we cannot inject the clock, so set the timestamps directly.)
var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
SetUpdatedAt(dataSource, older, t0);
SetUpdatedAt(dataSource, middle, t0.AddMinutes(10));
SetUpdatedAt(dataSource, newer, t0.AddMinutes(20));
// Cursor at the middle row's UpdatedAtUtc: inclusive lower bound, so
// middle + newer come back, older is excluded.
var result = await store.ReadChangedSinceAsync(t0.AddMinutes(10), batchSize: 100, CancellationToken.None);
Assert.Equal(2, result.Count);
Assert.Equal(middle, result[0].TrackedOperationId);
Assert.Equal(newer, result[1].TrackedOperationId);
Assert.True(result[0].UpdatedAtUtc <= result[1].UpdatedAtUtc);
}
[Fact]
public async Task ReadChangedSinceAsync_FromMinValue_ReturnsAllRows()
{
var (store, _) = CreateStore(nameof(ReadChangedSinceAsync_FromMinValue_ReturnsAllRows));
await using var _store = store;
await store.RecordEnqueueAsync(TrackedOperationId.New(), nameof(AuditKind.ApiCallCached), "A", null, null, null);
await store.RecordEnqueueAsync(TrackedOperationId.New(), nameof(AuditKind.ApiCallCached), "B", null, null, null);
var result = await store.ReadChangedSinceAsync(DateTime.MinValue, batchSize: 100, CancellationToken.None);
Assert.Equal(2, result.Count);
}
[Fact]
public async Task ReadChangedSinceAsync_IsBatchCapped()
{
var (store, dataSource) = CreateStore(nameof(ReadChangedSinceAsync_IsBatchCapped));
await using var _store = store;
var ids = new List<TrackedOperationId>();
var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
for (var i = 0; i < 5; i++)
{
var id = TrackedOperationId.New();
ids.Add(id);
await store.RecordEnqueueAsync(id, nameof(AuditKind.ApiCallCached), $"T{i}", null, null, null);
SetUpdatedAt(dataSource, id, t0.AddMinutes(i));
}
var result = await store.ReadChangedSinceAsync(DateTime.MinValue, batchSize: 3, CancellationToken.None);
// Capped to 3 — and the cap takes the OLDEST 3 (asc order) so the
// caller can advance the cursor monotonically across follow-up pulls.
Assert.Equal(3, result.Count);
Assert.Equal(ids[0], result[0].TrackedOperationId);
Assert.Equal(ids[1], result[1].TrackedOperationId);
Assert.Equal(ids[2], result[2].TrackedOperationId);
}
[Fact]
public async Task ReadChangedSinceAsync_MapsTrackingRowOntoSiteCallOperational()
{
var (store, _) = CreateStore(nameof(ReadChangedSinceAsync_MapsTrackingRowOntoSiteCallOperational));
await using var _store = store;
var apiId = TrackedOperationId.New();
var dbId = TrackedOperationId.New();
await store.RecordEnqueueAsync(apiId, nameof(AuditKind.ApiCallCached), "ERP.GetOrder", "inst-1", "ScriptActor:OnTick", "node-a");
await store.RecordEnqueueAsync(dbId, nameof(AuditKind.DbWriteCached), "Historian.Write", null, null, "node-b");
await store.RecordAttemptAsync(apiId, nameof(AuditStatus.Attempted), 2, "HTTP 503", 503);
await store.RecordTerminalAsync(dbId, nameof(AuditStatus.Parked), "max retries", null);
var result = await store.ReadChangedSinceAsync(DateTime.MinValue, batchSize: 100, CancellationToken.None);
var api = result.Single(r => r.TrackedOperationId == apiId);
var db = result.Single(r => r.TrackedOperationId == dbId);
// Kind → Channel projection.
Assert.Equal("ApiOutbound", api.Channel);
Assert.Equal("DbOutbound", db.Channel);
// TargetSummary → Target; SourceNode carried verbatim.
Assert.Equal("ERP.GetOrder", api.Target);
Assert.Equal("node-a", api.SourceNode);
Assert.Equal("node-b", db.SourceNode);
// Status / RetryCount / LastError / HttpStatus carried through.
Assert.Equal(nameof(AuditStatus.Attempted), api.Status);
Assert.Equal(2, api.RetryCount);
Assert.Equal("HTTP 503", api.LastError);
Assert.Equal(503, api.HttpStatus);
// SourceSite is left empty by the store (the site id is not a tracking
// column); the central client re-stamps it from the dialed siteId.
Assert.Equal(string.Empty, api.SourceSite);
// Terminal row carries TerminalAtUtc (UTC kind); active row leaves it null.
Assert.Null(api.TerminalAtUtc);
Assert.NotNull(db.TerminalAtUtc);
Assert.Equal(DateTimeKind.Utc, db.TerminalAtUtc!.Value.Kind);
// Timestamps round-trip as UTC.
Assert.Equal(DateTimeKind.Utc, api.CreatedAtUtc.Kind);
Assert.Equal(DateTimeKind.Utc, api.UpdatedAtUtc.Kind);
}
/// <summary>Directly sets a row's UpdatedAtUtc so cursor/ordering tests are deterministic.</summary>
private static void SetUpdatedAt(string dataSource, TrackedOperationId id, DateTime updatedAtUtc)
{
using var connection = OpenVerifierConnection(dataSource);
using var cmd = connection.CreateCommand();
cmd.CommandText = "UPDATE OperationTracking SET UpdatedAtUtc = $u WHERE TrackedOperationId = $id;";
cmd.Parameters.AddWithValue("$u", updatedAtUtc.ToString("o", System.Globalization.CultureInfo.InvariantCulture));
cmd.Parameters.AddWithValue("$id", id.ToString());
cmd.ExecuteNonQuery();
}
// ── SiteRuntime-024: read/write split + sync-safe Dispose ──────────────
[Fact]