fix(sql): chunk the key-value IN list at the dialect's bind ceiling (Driver.Sql-002, Driver.Sql.Dialect-002)
ISqlDialect gains MaxBindParameters — the per-command ceiling the seam was missing while modelling everything else that differs per backend: 2098 usable of SQL Server's 2100 (sp_executesql spends two), 1000 for Oracle's IN-list limit (ORA-01795, tighter than its 64k bind count), 65535 for the Postgres Bind message. SqlGroupPlanner.AddKeyValuePlans now emits several plans for one group when its distinct keys exceed that ceiling, instead of one statement the provider refuses after the connection opened — which left every tag in an EAV group publishing BadCommunicationError on every poll, permanently, and reported the database endpoint as down. Chunks share the group key; members follow their own key so a shared keyValue never splits. SqlPollReader is unchanged: it runs one command per plan and slices each plan's own Members back, proven by a fixture read at a ceiling of 1.
This commit is contained in:
Joseph Doherty
@@ -6,9 +6,10 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql;
|
||||
|
||||
/// <summary>
|
||||
/// The provider seam (design §2.2). ADO.NET's <see cref="System.Data.Common"/> base types abstract
|
||||
/// connections, commands, parameters and readers; they do <b>not</b> abstract the three things that differ
|
||||
/// per backend — <b>identifier quoting</b>, the <b>metadata-catalog SQL</b>, and <b>row-limit syntax</b>.
|
||||
/// Those live here, so no dialect assumption leaks into the reader, the poll planner, or the schema browser.
|
||||
/// connections, commands, parameters and readers; they do <b>not</b> abstract the four things that differ
|
||||
/// per backend — <b>identifier quoting</b>, the <b>metadata-catalog SQL</b>, <b>row-limit syntax</b>, and
|
||||
/// the <b>per-command bind-parameter ceiling</b>. Those live here, so no dialect assumption leaks into the
|
||||
/// reader, the poll planner, or the schema browser.
|
||||
/// <para><b>This interface is the driver's SQL-injection boundary.</b> Every runtime <em>value</em> is
|
||||
/// bound as a <see cref="DbParameter"/> and never reaches a command text. Identifiers cannot be
|
||||
/// parameterized in SQL, so the few that must appear as text are emitted through
|
||||
@@ -106,6 +107,24 @@ public interface ISqlDialect
|
||||
/// <param name="command">The command to configure.</param>
|
||||
void ConfigureCommand(DbCommand command);
|
||||
|
||||
/// <summary>
|
||||
/// The most bound parameters this backend accepts in <b>one</b> command — 2098 usable of SQL Server's
|
||||
/// 2100, 1000 entries in an Oracle <c>IN</c> expression list, 65535 on the Postgres wire protocol.
|
||||
/// Callers that build a variable-length parameter list must chunk at this number.
|
||||
/// <para><b>Why it belongs on the seam.</b> It differs per backend exactly as quoting, catalog SQL and
|
||||
/// row-limit syntax do, and a call site that guesses a provider limit it does not own is the defect
|
||||
/// this interface exists to prevent. Its absence was a shipped one: <c>SqlGroupPlanner</c> emitted one
|
||||
/// parameter per distinct <c>keyValue</c> with no bound, so a key-value (EAV) group above the cap was
|
||||
/// rejected by the provider <em>after</em> the connection opened and every tag in it published
|
||||
/// <c>BadCommunicationError</c> on every poll, permanently (Driver.Sql-002 /
|
||||
/// Driver.Sql.Dialect-002).</para>
|
||||
/// <para><b>The number is a ceiling on what the planner may emit, not the raw provider constant</b> —
|
||||
/// a dialect whose provider spends parameters of its own subtracts them here, so a caller filling the
|
||||
/// value exactly always produces an executable command. Must be at least 1; a dialect reporting less
|
||||
/// is rejected at the call site rather than looping or emitting an empty <c>IN ()</c>.</para>
|
||||
/// </summary>
|
||||
int MaxBindParameters { get; }
|
||||
|
||||
/// <summary>Cheapest statement that proves the connection is alive (<c>SELECT 1</c>; Oracle needs <c>FROM DUAL</c>).</summary>
|
||||
string LivenessSql { get; }
|
||||
|
||||
|
||||
@@ -103,6 +103,16 @@ public sealed class OracleDialect : ISqlDialect
|
||||
if (command is OracleCommand oracle) oracle.BindByName = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 1000 — <b>the tightest of the three ceilings, and the one that actually binds.</b> Oracle allows
|
||||
/// 64k bind variables per statement, but an <c>IN</c> <em>expression list</em> is limited to 1000
|
||||
/// entries (<c>ORA-01795</c>), and the key-value model's <c>IN (:p_k0, …)</c> list is the only
|
||||
/// variable-length parameter list this driver emits. The seam therefore reports the limit the caller
|
||||
/// will hit rather than the larger one it never reaches — a dialect that reported 64000 here would be
|
||||
/// technically accurate and would still produce a statement the server refuses.
|
||||
/// </summary>
|
||||
public int MaxBindParameters => 1000;
|
||||
|
||||
/// <summary>Oracle has no bare <c>SELECT</c>; every projection needs a table, and <c>DUAL</c> is the one-row one.</summary>
|
||||
public string LivenessSql => "SELECT 1 FROM DUAL";
|
||||
|
||||
|
||||
@@ -64,6 +64,14 @@ public sealed class PostgresDialect : ISqlDialect
|
||||
// Intentionally empty: see the summary.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 65535 — the extended-query protocol carries the parameter count as a signed/unsigned 16-bit field
|
||||
/// in <c>Bind</c>, so that is a protocol ceiling rather than a server setting, and Npgsql raises it
|
||||
/// client-side before a round-trip. Far above anything a poll group reaches in practice; it is here so
|
||||
/// the planner never has to special-case "this dialect has no meaningful limit".
|
||||
/// </summary>
|
||||
public int MaxBindParameters => 65535;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public string LivenessSql => "SELECT 1";
|
||||
|
||||
|
||||
@@ -59,6 +59,15 @@ public sealed class SqlServerDialect : ISqlDialect
|
||||
// Intentionally empty: see the summary.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 2098 — SQL Server's hard cap is <b>2100 parameters per statement</b>, and
|
||||
/// <c>Microsoft.Data.SqlClient</c> spends two of them itself: a parameterized command is executed
|
||||
/// through <c>sp_executesql</c>, whose first two arguments are the statement text and the parameter
|
||||
/// declaration list. A caller filling 2100 would therefore be rejected with
|
||||
/// <c>"The incoming request has too many parameters"</c> for a limit it appeared to respect.
|
||||
/// </summary>
|
||||
public int MaxBindParameters => 2098;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public string LivenessSql => "SELECT 1";
|
||||
|
||||
|
||||
@@ -7,6 +7,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql;
|
||||
/// <summary>
|
||||
/// Folds a set of <see cref="SqlTagDefinition"/>s into the minimum number of parameterized queries
|
||||
/// (design §3.6) — the driver's batching core. Pure and side-effect free: no connection, no I/O, no clock.
|
||||
/// <para><b>"Minimum" is bounded by the provider, not only by the group key.</b> A key-value group whose
|
||||
/// distinct keys exceed <see cref="ISqlDialect.MaxBindParameters"/> is emitted as several plans over the
|
||||
/// same group key rather than as one statement the backend refuses — see <see cref="AddKeyValuePlans"/>.</para>
|
||||
/// <para><b>The GroupKey is the correctness contract.</b> Two tags share a query <em>only</em> when every
|
||||
/// field that shapes the result set is identical. Too coarse and a tag silently reads a neighbour's cell;
|
||||
/// too fine and batching degenerates to one round-trip per tag. Both directions are pinned by
|
||||
@@ -31,14 +34,21 @@ public static class SqlGroupPlanner
|
||||
private const string RowSelectorBaseName = "w";
|
||||
|
||||
/// <summary>
|
||||
/// Groups <paramref name="tags"/> by source and emits one <see cref="SqlQueryPlan"/> per group.
|
||||
/// <para>Deterministic: groups appear in the input order of their first member, members keep their input
|
||||
/// order within a group, and parameters keep the first-appearance order of their distinct values — so a
|
||||
/// given tag list always yields byte-identical SQL.</para>
|
||||
/// Groups <paramref name="tags"/> by source and emits one <see cref="SqlQueryPlan"/> per group — or
|
||||
/// <b>several</b>, when a key-value group holds more distinct keys than the dialect's
|
||||
/// <see cref="ISqlDialect.MaxBindParameters"/> allows in one command (see
|
||||
/// <see cref="AddKeyValuePlans"/>). Plans from one group share that group's key.
|
||||
/// <para>Deterministic: groups appear in the input order of their first member, a split group's chunks
|
||||
/// stay adjacent and in key order, members keep their input order within a chunk, and parameters keep
|
||||
/// the first-appearance order of their distinct values — so a given tag list always yields
|
||||
/// byte-identical SQL.</para>
|
||||
/// </summary>
|
||||
/// <param name="tags">The tags to plan. An empty sequence yields no plans.</param>
|
||||
/// <param name="dialect">Supplies identifier quoting and the provider's row-limit syntax.</param>
|
||||
/// <returns>One plan per distinct group key.</returns>
|
||||
/// <param name="dialect">
|
||||
/// Supplies identifier quoting, the provider's row-limit syntax, and its per-command bind-parameter
|
||||
/// ceiling.
|
||||
/// </param>
|
||||
/// <returns>At least one plan per distinct group key; more when a key-value group had to be chunked.</returns>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="tags"/> or <paramref name="dialect"/> is null.</exception>
|
||||
/// <exception cref="ArgumentException">
|
||||
/// A definition is missing a field its model requires (e.g. a wide-row tag with no row selector), or an
|
||||
@@ -78,12 +88,18 @@ public static class SqlGroupPlanner
|
||||
foreach (var key in order)
|
||||
{
|
||||
var members = groups[key];
|
||||
plans.Add(members[0].Model switch
|
||||
switch (members[0].Model)
|
||||
{
|
||||
SqlTagModel.KeyValue => BuildKeyValuePlan(key, members, dialect),
|
||||
SqlTagModel.WideRow => BuildWideRowPlan(key, members, dialect),
|
||||
_ => throw UnsupportedModel(members[0]),
|
||||
});
|
||||
case SqlTagModel.KeyValue:
|
||||
// One group can yield SEVERAL plans: the IN list is chunked at the dialect's ceiling.
|
||||
AddKeyValuePlans(plans, key, members, dialect);
|
||||
break;
|
||||
case SqlTagModel.WideRow:
|
||||
plans.Add(BuildWideRowPlan(key, members, dialect));
|
||||
break;
|
||||
default:
|
||||
throw UnsupportedModel(members[0]);
|
||||
}
|
||||
}
|
||||
|
||||
return plans;
|
||||
@@ -119,10 +135,71 @@ public static class SqlGroupPlanner
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Emits <c>SELECT <key>, <value>[, <ts>] FROM <table> WHERE <key> IN (@k0..@kN)</c>.
|
||||
/// The key column is selected because the reader indexes rows by it to slice values back per member.
|
||||
/// Emits one key-value plan per <b>chunk</b> of the group's distinct keys, sized by
|
||||
/// <see cref="ISqlDialect.MaxBindParameters"/>. A group at or under the ceiling — every ordinary group
|
||||
/// — still yields exactly one plan.
|
||||
/// <para><b>Why chunk rather than let the provider refuse.</b> Every backend caps parameters per
|
||||
/// command (2100 on SQL Server, 1000 <c>IN</c>-list entries on Oracle), and a key-value (EAV) staging
|
||||
/// table is precisely the shape that attracts thousands of tags on <em>one</em> table — which the
|
||||
/// group-key rule folds into one group. Over the cap the provider rejects the statement <em>after</em>
|
||||
/// the connection opened, so every tag in the group publishes <c>BadCommunicationError</c> on every
|
||||
/// poll, permanently, and the driver reports a communication fault against a database that is
|
||||
/// answering (Driver.Sql-002).</para>
|
||||
/// <para><b>Chunks share the group key, and no consumer needs to change.</b> The reader executes one
|
||||
/// command per plan and slices each plan's own <see cref="SqlQueryPlan.Members"/> back, so several
|
||||
/// plans over one group key cost extra round-trips and nothing else. The alternative — a deploy-time
|
||||
/// tag-count gate — would refuse a configuration the driver can serve perfectly well.</para>
|
||||
/// <para><b>Members follow their key, never their position.</b> Two tags authored on the same
|
||||
/// <c>keyValue</c> land in the same chunk (one bound parameter, both fed); a member placed in a chunk
|
||||
/// that does not bind its key would read a result set that cannot contain its row and publish
|
||||
/// <c>BadNoData</c> forever.</para>
|
||||
/// </summary>
|
||||
private static void AddKeyValuePlans(
|
||||
List<SqlQueryPlan> plans, object groupKey, List<SqlTagDefinition> members, ISqlDialect dialect)
|
||||
{
|
||||
var ceiling = dialect.MaxBindParameters;
|
||||
if (ceiling < 1)
|
||||
{
|
||||
// Not a silent clamp: chunking to zero would loop forever, and a one-per-chunk fallback would
|
||||
// hide a broken dialect behind a poll that suddenly costs one round-trip per tag.
|
||||
throw new ArgumentException(
|
||||
$"The {dialect.Provider} dialect reports a per-command bind-parameter ceiling of {ceiling}; " +
|
||||
"it must be at least 1.", nameof(dialect));
|
||||
}
|
||||
|
||||
// Distinct keys in first-appearance order decide the chunk boundaries; every member then follows its
|
||||
// own key. Both halves keep the planner's determinism: a given tag list yields byte-identical SQL.
|
||||
var chunkOfKey = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
foreach (var member in members)
|
||||
{
|
||||
var keyValue = member.KeyValue
|
||||
?? throw MissingField(nameof(SqlTagDefinition.KeyValue), member);
|
||||
if (!chunkOfKey.ContainsKey(keyValue)) chunkOfKey[keyValue] = chunkOfKey.Count / ceiling;
|
||||
}
|
||||
|
||||
var chunkCount = ((chunkOfKey.Count - 1) / ceiling) + 1;
|
||||
if (chunkCount == 1)
|
||||
{
|
||||
plans.Add(BuildKeyValuePlan(groupKey, members, dialect));
|
||||
return;
|
||||
}
|
||||
|
||||
var chunks = new List<SqlTagDefinition>[chunkCount];
|
||||
for (var c = 0; c < chunkCount; c++) chunks[c] = [];
|
||||
foreach (var member in members) chunks[chunkOfKey[member.KeyValue!]].Add(member);
|
||||
|
||||
foreach (var chunk in chunks) plans.Add(BuildKeyValuePlan(groupKey, chunk, dialect));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Emits <c>SELECT <key>, <value>[, <ts>] FROM <table> WHERE <key> IN (@k0..@kN)</c>
|
||||
/// for one chunk of a group. The key column is selected because the reader indexes rows by it to slice
|
||||
/// values back per member.
|
||||
/// <para>Keys are de-duplicated (ordinal) before binding: two tags authored against the same
|
||||
/// <c>keyValue</c> bind one parameter but stay two members, so both nodes are fed from the one row.</para>
|
||||
/// <para>Markers are numbered from <c>k0</c> within the chunk — each chunk is its own command, so
|
||||
/// carrying a group-wide index forward would leave a three-parameter command spelling
|
||||
/// <c>IN (@k3, @k4, @k5)</c>.</para>
|
||||
/// </summary>
|
||||
private static SqlQueryPlan BuildKeyValuePlan(
|
||||
object groupKey, List<SqlTagDefinition> members, ISqlDialect dialect)
|
||||
|
||||
@@ -27,6 +27,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql;
|
||||
/// <param name="GroupKey">
|
||||
/// The opaque equality key that decided this grouping — a value tuple, so two plans compare equal exactly
|
||||
/// when they would have folded together. Exposed for diagnostics and plan caching, never for parsing.
|
||||
/// <para><b>Not unique across plans.</b> A key-value group holding more distinct keys than the dialect's
|
||||
/// <see cref="ISqlDialect.MaxBindParameters"/> is emitted as several plans carrying <em>this same key</em>
|
||||
/// — the chunking is an execution detail of one group, not a regrouping.</para>
|
||||
/// </param>
|
||||
/// <param name="SqlText">The single parameterized statement to execute for this group.</param>
|
||||
/// <param name="ParameterNames">
|
||||
|
||||
Reference in New Issue
Block a user