dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sql): chunk the key-value IN list at the dialect's bind ceiling (Driver.Sql-002, Driver.Sql.Dialect-002)

Browse Source 
ISqlDialect gains MaxBindParameters — the per-command ceiling the seam was
missing while modelling everything else that differs per backend: 2098 usable
of SQL Server's 2100 (sp_executesql spends two), 1000 for Oracle's IN-list
limit (ORA-01795, tighter than its 64k bind count), 65535 for the Postgres
Bind message.

SqlGroupPlanner.AddKeyValuePlans now emits several plans for one group when
its distinct keys exceed that ceiling, instead of one statement the provider
refuses after the connection opened — which left every tag in an EAV group
publishing BadCommunicationError on every poll, permanently, and reported the
database endpoint as down. Chunks share the group key; members follow their
own key so a shared keyValue never splits. SqlPollReader is unchanged: it runs
one command per plan and slices each plan's own Members back, proven by a
fixture read at a ceiling of 1.
This commit is contained in:
Joseph Doherty
2026-08-10 08:02:44 -04:00
parent 74594e6083
commit 886ce1b87c
14 changed files with 509 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs
+90 -13
View File
@@ -7,6 +7,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql;
/// <summary>
/// Folds a set of <see cref="SqlTagDefinition"/>s into the minimum number of parameterized queries
/// (design §3.6) — the driver's batching core. Pure and side-effect free: no connection, no I/O, no clock.
/// <para><b>"Minimum" is bounded by the provider, not only by the group key.</b> A key-value group whose
/// distinct keys exceed <see cref="ISqlDialect.MaxBindParameters"/> is emitted as several plans over the
/// same group key rather than as one statement the backend refuses — see <see cref="AddKeyValuePlans"/>.</para>
/// <para><b>The GroupKey is the correctness contract.</b> Two tags share a query <em>only</em> when every
/// field that shapes the result set is identical. Too coarse and a tag silently reads a neighbour's cell;
/// too fine and batching degenerates to one round-trip per tag. Both directions are pinned by
@@ -31,14 +34,21 @@ public static class SqlGroupPlanner
private const string RowSelectorBaseName = "w";
/// <summary>
/// Groups <paramref name="tags"/> by source and emits one <see cref="SqlQueryPlan"/> per group.
/// <para>Deterministic: groups appear in the input order of their first member, members keep their input
/// order within a group, and parameters keep the first-appearance order of their distinct values — so a
/// given tag list always yields byte-identical SQL.</para>
/// Groups <paramref name="tags"/> by source and emits one <see cref="SqlQueryPlan"/> per group — or
/// <b>several</b>, when a key-value group holds more distinct keys than the dialect's
/// <see cref="ISqlDialect.MaxBindParameters"/> allows in one command (see
/// <see cref="AddKeyValuePlans"/>). Plans from one group share that group's key.
/// <para>Deterministic: groups appear in the input order of their first member, a split group's chunks
/// stay adjacent and in key order, members keep their input order within a chunk, and parameters keep
/// the first-appearance order of their distinct values — so a given tag list always yields
/// byte-identical SQL.</para>
/// </summary>
/// <param name="tags">The tags to plan. An empty sequence yields no plans.</param>
/// <param name="dialect">Supplies identifier quoting and the provider's row-limit syntax.</param>
/// <returns>One plan per distinct group key.</returns>
/// <param name="dialect">
/// Supplies identifier quoting, the provider's row-limit syntax, and its per-command bind-parameter
/// ceiling.
/// </param>
/// <returns>At least one plan per distinct group key; more when a key-value group had to be chunked.</returns>
/// <exception cref="ArgumentNullException"><paramref name="tags"/> or <paramref name="dialect"/> is null.</exception>
/// <exception cref="ArgumentException">
/// A definition is missing a field its model requires (e.g. a wide-row tag with no row selector), or an
@@ -78,12 +88,18 @@ public static class SqlGroupPlanner
foreach (var key in order)
{
var members = groups[key];
plans.Add(members[0].Model switch
switch (members[0].Model)
{
SqlTagModel.KeyValue => BuildKeyValuePlan(key, members, dialect),
SqlTagModel.WideRow => BuildWideRowPlan(key, members, dialect),
_ => throw UnsupportedModel(members[0]),
});
case SqlTagModel.KeyValue:
// One group can yield SEVERAL plans: the IN list is chunked at the dialect's ceiling.
AddKeyValuePlans(plans, key, members, dialect);
break;
case SqlTagModel.WideRow:
plans.Add(BuildWideRowPlan(key, members, dialect));
break;
default:
throw UnsupportedModel(members[0]);
}
}
return plans;
@@ -119,10 +135,71 @@ public static class SqlGroupPlanner
};
/// <summary>
/// Emits <c>SELECT &lt;key&gt;, &lt;value&gt;[, &lt;ts&gt;] FROM &lt;table&gt; WHERE &lt;key&gt; IN (@k0..@kN)</c>.
/// The key column is selected because the reader indexes rows by it to slice values back per member.
/// Emits one key-value plan per <b>chunk</b> of the group's distinct keys, sized by
/// <see cref="ISqlDialect.MaxBindParameters"/>. A group at or under the ceiling — every ordinary group
/// — still yields exactly one plan.
/// <para><b>Why chunk rather than let the provider refuse.</b> Every backend caps parameters per
/// command (2100 on SQL Server, 1000 <c>IN</c>-list entries on Oracle), and a key-value (EAV) staging
/// table is precisely the shape that attracts thousands of tags on <em>one</em> table — which the
/// group-key rule folds into one group. Over the cap the provider rejects the statement <em>after</em>
/// the connection opened, so every tag in the group publishes <c>BadCommunicationError</c> on every
/// poll, permanently, and the driver reports a communication fault against a database that is
/// answering (Driver.Sql-002).</para>
/// <para><b>Chunks share the group key, and no consumer needs to change.</b> The reader executes one
/// command per plan and slices each plan's own <see cref="SqlQueryPlan.Members"/> back, so several
/// plans over one group key cost extra round-trips and nothing else. The alternative — a deploy-time
/// tag-count gate — would refuse a configuration the driver can serve perfectly well.</para>
/// <para><b>Members follow their key, never their position.</b> Two tags authored on the same
/// <c>keyValue</c> land in the same chunk (one bound parameter, both fed); a member placed in a chunk
/// that does not bind its key would read a result set that cannot contain its row and publish
/// <c>BadNoData</c> forever.</para>
/// </summary>
private static void AddKeyValuePlans(
List<SqlQueryPlan> plans, object groupKey, List<SqlTagDefinition> members, ISqlDialect dialect)
{
var ceiling = dialect.MaxBindParameters;
if (ceiling < 1)
{
// Not a silent clamp: chunking to zero would loop forever, and a one-per-chunk fallback would
// hide a broken dialect behind a poll that suddenly costs one round-trip per tag.
throw new ArgumentException(
$"The {dialect.Provider} dialect reports a per-command bind-parameter ceiling of {ceiling}; " +
"it must be at least 1.", nameof(dialect));
}
// Distinct keys in first-appearance order decide the chunk boundaries; every member then follows its
// own key. Both halves keep the planner's determinism: a given tag list yields byte-identical SQL.
var chunkOfKey = new Dictionary<string, int>(StringComparer.Ordinal);
foreach (var member in members)
{
var keyValue = member.KeyValue
?? throw MissingField(nameof(SqlTagDefinition.KeyValue), member);
if (!chunkOfKey.ContainsKey(keyValue)) chunkOfKey[keyValue] = chunkOfKey.Count / ceiling;
}
var chunkCount = ((chunkOfKey.Count - 1) / ceiling) + 1;
if (chunkCount == 1)
{
plans.Add(BuildKeyValuePlan(groupKey, members, dialect));
return;
}
var chunks = new List<SqlTagDefinition>[chunkCount];
for (var c = 0; c < chunkCount; c++) chunks[c] = [];
foreach (var member in members) chunks[chunkOfKey[member.KeyValue!]].Add(member);
foreach (var chunk in chunks) plans.Add(BuildKeyValuePlan(groupKey, chunk, dialect));
}
/// <summary>
/// Emits <c>SELECT &lt;key&gt;, &lt;value&gt;[, &lt;ts&gt;] FROM &lt;table&gt; WHERE &lt;key&gt; IN (@k0..@kN)</c>
/// for one chunk of a group. The key column is selected because the reader indexes rows by it to slice
/// values back per member.
/// <para>Keys are de-duplicated (ordinal) before binding: two tags authored against the same
/// <c>keyValue</c> bind one parameter but stay two members, so both nodes are fed from the one row.</para>
/// <para>Markers are numbered from <c>k0</c> within the chunk — each chunk is its own command, so
/// carrying a group-wide index forward would leave a three-parameter command spelling
/// <c>IN (@k3, @k4, @k5)</c>.</para>
/// </summary>
private static SqlQueryPlan BuildKeyValuePlan(
object groupKey, List<SqlTagDefinition> members, ISqlDialect dialect)