merge: SQL dialect+chunking review findings resolved (Driver.Sql-002, Driver.Sql.Dialect-001/-002/-003)

This commit is contained in:
Joseph Doherty
2026-08-10 08:06:13 -04:00
16 changed files with 648 additions and 37 deletions
@@ -6,9 +6,10 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql;
/// <summary>
/// The provider seam (design §2.2). ADO.NET's <see cref="System.Data.Common"/> base types abstract
/// connections, commands, parameters and readers; they do <b>not</b> abstract the three things that differ
/// per backend — <b>identifier quoting</b>, the <b>metadata-catalog SQL</b>, and <b>row-limit syntax</b>.
/// Those live here, so no dialect assumption leaks into the reader, the poll planner, or the schema browser.
/// connections, commands, parameters and readers; they do <b>not</b> abstract the four things that differ
/// per backend — <b>identifier quoting</b>, the <b>metadata-catalog SQL</b>, <b>row-limit syntax</b>, and
/// the <b>per-command bind-parameter ceiling</b>. Those live here, so no dialect assumption leaks into the
/// reader, the poll planner, or the schema browser.
/// <para><b>This interface is the driver's SQL-injection boundary.</b> Every runtime <em>value</em> is
/// bound as a <see cref="DbParameter"/> and never reaches a command text. Identifiers cannot be
/// parameterized in SQL, so the few that must appear as text are emitted through
@@ -106,6 +107,24 @@ public interface ISqlDialect
/// <param name="command">The command to configure.</param>
void ConfigureCommand(DbCommand command);
/// <summary>
/// The most bound parameters this backend accepts in <b>one</b> command — 2098 usable of SQL Server's
/// 2100, 1000 entries in an Oracle <c>IN</c> expression list, 65535 on the Postgres wire protocol.
/// Callers that build a variable-length parameter list must chunk at this number.
/// <para><b>Why it belongs on the seam.</b> It differs per backend exactly as quoting, catalog SQL and
/// row-limit syntax do, and a call site that guesses a provider limit it does not own is the defect
/// this interface exists to prevent. Its absence was a shipped one: <c>SqlGroupPlanner</c> emitted one
/// parameter per distinct <c>keyValue</c> with no bound, so a key-value (EAV) group above the cap was
/// rejected by the provider <em>after</em> the connection opened and every tag in it published
/// <c>BadCommunicationError</c> on every poll, permanently (Driver.Sql-002 /
/// Driver.Sql.Dialect-002).</para>
/// <para><b>The number is a ceiling on what the planner may emit, not the raw provider constant</b> —
/// a dialect whose provider spends parameters of its own subtracts them here, so a caller filling the
/// value exactly always produces an executable command. Must be at least 1; a dialect reporting less
/// is rejected at the call site rather than looping or emitting an empty <c>IN ()</c>.</para>
/// </summary>
int MaxBindParameters { get; }
/// <summary>Cheapest statement that proves the connection is alive (<c>SELECT 1</c>; Oracle needs <c>FROM DUAL</c>).</summary>
string LivenessSql { get; }
@@ -1,5 +1,6 @@
using System.Data.Common;
using System.Globalization;
using System.Text;
using Oracle.ManagedDataAccess.Client;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Sql.Contracts;
@@ -24,6 +25,9 @@ public sealed class OracleDialect : ISqlDialect
/// <summary>
/// Oracle 12.2+'s identifier ceiling. Enforced because a longer string cannot name a real catalog
/// object — it is either a config mistake or padding, and rejecting it keeps the quoted output bounded.
/// <para><b>The unit is BYTES in the database character set, not characters</b> — see
/// <see cref="QuoteIdentifier"/>. Practically that means UTF-8 (<c>AL32UTF8</c>) on any modern estate,
/// so a 65-character Cyrillic name is already over the limit.</para>
/// </summary>
internal const int MaxIdentifierLength = 128;
@@ -99,6 +103,16 @@ public sealed class OracleDialect : ISqlDialect
if (command is OracleCommand oracle) oracle.BindByName = true;
}
/// <summary>
/// 1000 — <b>the tightest of the three ceilings, and the one that actually binds.</b> Oracle allows
/// 64k bind variables per statement, but an <c>IN</c> <em>expression list</em> is limited to 1000
/// entries (<c>ORA-01795</c>), and the key-value model's <c>IN (:p_k0, …)</c> list is the only
/// variable-length parameter list this driver emits. The seam therefore reports the limit the caller
/// will hit rather than the larger one it never reaches — a dialect that reported 64000 here would be
/// technically accurate and would still produce a statement the server refuses.
/// </summary>
public int MaxBindParameters => 1000;
/// <summary>Oracle has no bare <c>SELECT</c>; every projection needs a table, and <c>DUAL</c> is the one-row one.</summary>
public string LivenessSql => "SELECT 1 FROM DUAL";
@@ -127,10 +141,26 @@ public sealed class OracleDialect : ISqlDialect
/// user), aliased to the <c>TABLE_SCHEMA</c> column name the catalog readers project by.
/// </summary>
/// <remarks>
/// Broader than the SQL Server dialect's <c>DISTINCT TABLE_SCHEMA FROM INFORMATION_SCHEMA.TABLES</c>,
/// which lists only schemas that own an object: <c>ALL_USERS</c> includes empty ones. Harmless here —
/// the list is used to resolve an authored schema name to the catalog's own spelling, and a schema
/// holding no tables simply resolves and then matches no relation.
/// <para>Broader than the SQL Server dialect's <c>DISTINCT TABLE_SCHEMA FROM
/// INFORMATION_SCHEMA.TABLES</c>, which lists only schemas that own an object: <c>ALL_USERS</c>
/// includes empty ones. Harmless for the <b>catalog gate</b> — the list is used to resolve an authored
/// schema name to the catalog's own spelling, and a schema holding no tables simply resolves and then
/// matches no relation.</para>
/// <para>⚠️ <b>It is NOT harmless for the browse tree, and that is a known, accepted trade rather than
/// an oversight (Driver.Sql.Dialect-003).</b> This query has two consumers, and only the gate wants the
/// broad list: <c>SqlBrowseSession.RootAsync</c> renders every row as an expandable schema folder, so
/// on a real Oracle estate the operator is offered every database <em>user</em> — hundreds of them,
/// nearly all empty, with nothing in the tree distinguishing the ones that hold a relation until each
/// is expanded.</para>
/// <para><b>The narrower query is
/// <c>SELECT DISTINCT owner FROM ALL_TABLES UNION SELECT DISTINCT owner FROM ALL_VIEWS</c></b>, which
/// would match what the other two dialects list and what the browse tree wants. It is deliberately not
/// taken here because it also <em>narrows the gate</em>: a tag naming an empty schema today resolves and
/// then fails at the table level, and afterwards would fail at the schema level with a different
/// message — a behaviour change on the poll path that can only be validated against a real Oracle
/// server (<c>OracleReadTests</c>), never offline, for the same reason the <c>p_</c> bind-prefix defect
/// was invisible until it ran. Make the swap when an Oracle estate is available to gate it on, and
/// change both consumers' expectations in the same commit.</para>
/// </remarks>
public string ListSchemasSql =>
"SELECT username AS TABLE_SCHEMA FROM ALL_USERS ORDER BY username";
@@ -187,8 +217,14 @@ public sealed class OracleDialect : ISqlDialect
/// (<c>ALL_TAB_COLUMNS.column_name</c>, i.e. what the server actually stored) before anything reaches
/// this method — an operator who authored <c>speed</c> against an Oracle <c>SPEED</c> column resolves
/// to <c>SPEED</c> at Initialize, and it is <c>SPEED</c> that gets quoted.</para>
/// <para><b>The length ceiling is measured in UTF-8 bytes, not characters.</b> Oracle's 128 bounds the
/// name in the <em>database character set</em> (<c>AL32UTF8</c> on any modern estate), so counting
/// UTF-16 code units would accept a 65-character Cyrillic name — 130 bytes — and let it fail at the
/// server with <c>ORA-00972</c> rather than here, where the message can name the tag.
/// <c>SqlServerDialect</c> deliberately does <b>not</b> do this: T-SQL's <c>sysname</c> is
/// <c>nvarchar(128)</c>, a genuine 128-character limit.</para>
/// <para><b>Rejects</b> (all as <see cref="ArgumentException"/>): null / empty / all-whitespace; longer
/// than <see cref="MaxIdentifierLength"/>; any control character (which covers embedded NUL, and the
/// than <see cref="MaxIdentifierLength"/> <b>bytes</b>; any control character (which covers embedded NUL, and the
/// C0/C1 ranges that can truncate or corrupt a logged/audited statement); and any Unicode
/// <see cref="UnicodeCategory.Format"/> character — bidi overrides, zero-width spaces, soft hyphen,
/// BOM. Those last cannot escape the quotes, but they spoof the <em>rendered</em> text of a log line or
@@ -209,9 +245,13 @@ public sealed class OracleDialect : ISqlDialect
throw new ArgumentException(
"A SQL identifier must be a non-empty, non-whitespace name.", nameof(ident));
if (ident.Length > MaxIdentifierLength)
// Byte count, not ident.Length: Oracle's 128 is measured in the database character set, so a
// 65-character Cyrillic name is 130 UTF-8 bytes and the server answers ORA-00972. Counting UTF-16
// code units would send it anyway, failing at the server for a reason this dialect could have named.
var byteLength = Encoding.UTF8.GetByteCount(ident);
if (byteLength > MaxIdentifierLength)
throw new ArgumentException(
$"An Oracle identifier may not exceed {MaxIdentifierLength} characters (got {ident.Length}).",
$"An Oracle identifier may not exceed {MaxIdentifierLength} bytes (got {byteLength}).",
nameof(ident));
foreach (var ch in ident)
@@ -1,5 +1,6 @@
using System.Data.Common;
using System.Globalization;
using System.Text;
using Npgsql;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Sql.Contracts;
@@ -20,6 +21,9 @@ public sealed class PostgresDialect : ISqlDialect
/// PostgreSQL's identifier ceiling — <c>NAMEDATALEN - 1</c>, 63 by default. Enforced because the
/// server <b>silently truncates</b> anything longer, so a 70-character authored name would resolve to
/// a different object than it spells; rejecting it surfaces the config mistake instead.
/// <para><b>The unit is BYTES in the database encoding, not characters</b> — see
/// <see cref="QuoteIdentifier"/>. Practically that means UTF-8 on any modern estate, so a 32-character
/// Cyrillic name is already over the limit.</para>
/// </summary>
internal const int MaxIdentifierLength = 63;
@@ -60,6 +64,14 @@ public sealed class PostgresDialect : ISqlDialect
// Intentionally empty: see the summary.
}
/// <summary>
/// 65535 — the extended-query protocol carries the parameter count as a signed/unsigned 16-bit field
/// in <c>Bind</c>, so that is a protocol ceiling rather than a server setting, and Npgsql raises it
/// client-side before a round-trip. Far above anything a poll group reaches in practice; it is here so
/// the planner never has to special-case "this dialect has no meaningful limit".
/// </summary>
public int MaxBindParameters => 65535;
/// <inheritdoc/>
public string LivenessSql => "SELECT 1";
@@ -149,12 +161,19 @@ public sealed class PostgresDialect : ISqlDialect
/// <para><b>Rejects</b> (all as <see cref="ArgumentException"/>, mirroring
/// <c>SqlServerDialect.QuoteIdentifier</c> so a test written against one dialect fails the same way
/// against the other): null / empty / all-whitespace; longer than
/// <see cref="MaxIdentifierLength"/>; any control character (which covers embedded NUL and the C0/C1
/// <see cref="MaxIdentifierLength"/> <b>bytes</b>; any control character (which covers embedded NUL and the C0/C1
/// ranges that can truncate or corrupt a logged/audited statement); and any Unicode
/// <see cref="UnicodeCategory.Format"/> character — bidi overrides, zero-width spaces, soft hyphen,
/// BOM. Those last cannot escape the quotes, but they spoof the <em>rendered</em> text of a log line
/// or an AdminUI label while comparing byte-different from the real catalog name (the Trojan-Source
/// class, CVE-2021-42574).</para>
/// <para><b>The length ceiling is measured in UTF-8 bytes, not characters</b> — <c>NAMEDATALEN</c>
/// bounds the stored name in the database encoding, and every modern estate is UTF-8. Counting UTF-16
/// code units (<c>ident.Length</c>) would accept a 32-character Cyrillic name, which is 64 bytes and
/// is therefore truncated server-side to a name that denotes a <em>different</em> object — precisely
/// the silent-truncation failure this rule exists to prevent. <c>SqlServerDialect</c> deliberately does
/// <b>not</b> do this: T-SQL's <c>sysname</c> is <c>nvarchar(128)</c>, a genuine 128-character limit,
/// so counting characters is correct there and counting bytes would reject legal names.</para>
/// <para>NUL deserves its own note here: Postgres rejects a NUL byte anywhere in a query string
/// outright, so passing one through would fail the whole statement rather than just the one
/// identifier.</para>
@@ -175,9 +194,13 @@ public sealed class PostgresDialect : ISqlDialect
throw new ArgumentException(
"A SQL identifier must be a non-empty, non-whitespace name.", nameof(ident));
if (ident.Length > MaxIdentifierLength)
// Byte count, not ident.Length: NAMEDATALEN bounds the stored name in the database encoding, so a
// 32-character Cyrillic name is 64 UTF-8 bytes and the server truncates it — the exact silent
// wrong-object outcome this check exists to prevent, and one a UTF-16 code-unit count waves through.
var byteLength = Encoding.UTF8.GetByteCount(ident);
if (byteLength > MaxIdentifierLength)
throw new ArgumentException(
$"A PostgreSQL identifier may not exceed {MaxIdentifierLength} characters (got {ident.Length}).",
$"A PostgreSQL identifier may not exceed {MaxIdentifierLength} bytes (got {byteLength}).",
nameof(ident));
foreach (var ch in ident)
@@ -18,6 +18,10 @@ public sealed class SqlServerDialect : ISqlDialect
/// T-SQL's regular-identifier ceiling (<c>sysname</c> = <c>nvarchar(128)</c>). Enforced because a
/// longer string cannot name a real catalog object — it is either a config mistake or padding, and
/// rejecting it keeps the quoted output bounded.
/// <para><b>The unit here is genuinely CHARACTERS</b> (UTF-16 code units, which is what
/// <c>nvarchar(128)</c> counts), deliberately unlike <c>PostgresDialect</c> and <c>OracleDialect</c>,
/// whose ceilings are byte limits in the database encoding. Measuring bytes here would reject a legal
/// 128-character non-ASCII name.</para>
/// </summary>
internal const int MaxIdentifierLength = 128;
@@ -55,6 +59,15 @@ public sealed class SqlServerDialect : ISqlDialect
// Intentionally empty: see the summary.
}
/// <summary>
/// 2098 — SQL Server's hard cap is <b>2100 parameters per statement</b>, and
/// <c>Microsoft.Data.SqlClient</c> spends two of them itself: a parameterized command is executed
/// through <c>sp_executesql</c>, whose first two arguments are the statement text and the parameter
/// declaration list. A caller filling 2100 would therefore be rejected with
/// <c>"The incoming request has too many parameters"</c> for a limit it appeared to respect.
/// </summary>
public int MaxBindParameters => 2098;
/// <inheritdoc/>
public string LivenessSql => "SELECT 1";
@@ -7,6 +7,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql;
/// <summary>
/// Folds a set of <see cref="SqlTagDefinition"/>s into the minimum number of parameterized queries
/// (design §3.6) — the driver's batching core. Pure and side-effect free: no connection, no I/O, no clock.
/// <para><b>"Minimum" is bounded by the provider, not only by the group key.</b> A key-value group whose
/// distinct keys exceed <see cref="ISqlDialect.MaxBindParameters"/> is emitted as several plans over the
/// same group key rather than as one statement the backend refuses — see <see cref="AddKeyValuePlans"/>.</para>
/// <para><b>The GroupKey is the correctness contract.</b> Two tags share a query <em>only</em> when every
/// field that shapes the result set is identical. Too coarse and a tag silently reads a neighbour's cell;
/// too fine and batching degenerates to one round-trip per tag. Both directions are pinned by
@@ -31,14 +34,21 @@ public static class SqlGroupPlanner
private const string RowSelectorBaseName = "w";
/// <summary>
/// Groups <paramref name="tags"/> by source and emits one <see cref="SqlQueryPlan"/> per group.
/// <para>Deterministic: groups appear in the input order of their first member, members keep their input
/// order within a group, and parameters keep the first-appearance order of their distinct values — so a
/// given tag list always yields byte-identical SQL.</para>
/// Groups <paramref name="tags"/> by source and emits one <see cref="SqlQueryPlan"/> per group — or
/// <b>several</b>, when a key-value group holds more distinct keys than the dialect's
/// <see cref="ISqlDialect.MaxBindParameters"/> allows in one command (see
/// <see cref="AddKeyValuePlans"/>). Plans from one group share that group's key.
/// <para>Deterministic: groups appear in the input order of their first member, a split group's chunks
/// stay adjacent and in key order, members keep their input order within a chunk, and parameters keep
/// the first-appearance order of their distinct values — so a given tag list always yields
/// byte-identical SQL.</para>
/// </summary>
/// <param name="tags">The tags to plan. An empty sequence yields no plans.</param>
/// <param name="dialect">Supplies identifier quoting and the provider's row-limit syntax.</param>
/// <returns>One plan per distinct group key.</returns>
/// <param name="dialect">
/// Supplies identifier quoting, the provider's row-limit syntax, and its per-command bind-parameter
/// ceiling.
/// </param>
/// <returns>At least one plan per distinct group key; more when a key-value group had to be chunked.</returns>
/// <exception cref="ArgumentNullException"><paramref name="tags"/> or <paramref name="dialect"/> is null.</exception>
/// <exception cref="ArgumentException">
/// A definition is missing a field its model requires (e.g. a wide-row tag with no row selector), or an
@@ -78,12 +88,18 @@ public static class SqlGroupPlanner
foreach (var key in order)
{
var members = groups[key];
plans.Add(members[0].Model switch
switch (members[0].Model)
{
SqlTagModel.KeyValue => BuildKeyValuePlan(key, members, dialect),
SqlTagModel.WideRow => BuildWideRowPlan(key, members, dialect),
_ => throw UnsupportedModel(members[0]),
});
case SqlTagModel.KeyValue:
// One group can yield SEVERAL plans: the IN list is chunked at the dialect's ceiling.
AddKeyValuePlans(plans, key, members, dialect);
break;
case SqlTagModel.WideRow:
plans.Add(BuildWideRowPlan(key, members, dialect));
break;
default:
throw UnsupportedModel(members[0]);
}
}
return plans;
@@ -119,10 +135,71 @@ public static class SqlGroupPlanner
};
/// <summary>
/// Emits <c>SELECT &lt;key&gt;, &lt;value&gt;[, &lt;ts&gt;] FROM &lt;table&gt; WHERE &lt;key&gt; IN (@k0..@kN)</c>.
/// The key column is selected because the reader indexes rows by it to slice values back per member.
/// Emits one key-value plan per <b>chunk</b> of the group's distinct keys, sized by
/// <see cref="ISqlDialect.MaxBindParameters"/>. A group at or under the ceiling — every ordinary group
/// — still yields exactly one plan.
/// <para><b>Why chunk rather than let the provider refuse.</b> Every backend caps parameters per
/// command (2100 on SQL Server, 1000 <c>IN</c>-list entries on Oracle), and a key-value (EAV) staging
/// table is precisely the shape that attracts thousands of tags on <em>one</em> table — which the
/// group-key rule folds into one group. Over the cap the provider rejects the statement <em>after</em>
/// the connection opened, so every tag in the group publishes <c>BadCommunicationError</c> on every
/// poll, permanently, and the driver reports a communication fault against a database that is
/// answering (Driver.Sql-002).</para>
/// <para><b>Chunks share the group key, and no consumer needs to change.</b> The reader executes one
/// command per plan and slices each plan's own <see cref="SqlQueryPlan.Members"/> back, so several
/// plans over one group key cost extra round-trips and nothing else. The alternative — a deploy-time
/// tag-count gate — would refuse a configuration the driver can serve perfectly well.</para>
/// <para><b>Members follow their key, never their position.</b> Two tags authored on the same
/// <c>keyValue</c> land in the same chunk (one bound parameter, both fed); a member placed in a chunk
/// that does not bind its key would read a result set that cannot contain its row and publish
/// <c>BadNoData</c> forever.</para>
/// </summary>
private static void AddKeyValuePlans(
List<SqlQueryPlan> plans, object groupKey, List<SqlTagDefinition> members, ISqlDialect dialect)
{
var ceiling = dialect.MaxBindParameters;
if (ceiling < 1)
{
// Not a silent clamp: chunking to zero would loop forever, and a one-per-chunk fallback would
// hide a broken dialect behind a poll that suddenly costs one round-trip per tag.
throw new ArgumentException(
$"The {dialect.Provider} dialect reports a per-command bind-parameter ceiling of {ceiling}; " +
"it must be at least 1.", nameof(dialect));
}
// Distinct keys in first-appearance order decide the chunk boundaries; every member then follows its
// own key. Both halves keep the planner's determinism: a given tag list yields byte-identical SQL.
var chunkOfKey = new Dictionary<string, int>(StringComparer.Ordinal);
foreach (var member in members)
{
var keyValue = member.KeyValue
?? throw MissingField(nameof(SqlTagDefinition.KeyValue), member);
if (!chunkOfKey.ContainsKey(keyValue)) chunkOfKey[keyValue] = chunkOfKey.Count / ceiling;
}
var chunkCount = ((chunkOfKey.Count - 1) / ceiling) + 1;
if (chunkCount == 1)
{
plans.Add(BuildKeyValuePlan(groupKey, members, dialect));
return;
}
var chunks = new List<SqlTagDefinition>[chunkCount];
for (var c = 0; c < chunkCount; c++) chunks[c] = [];
foreach (var member in members) chunks[chunkOfKey[member.KeyValue!]].Add(member);
foreach (var chunk in chunks) plans.Add(BuildKeyValuePlan(groupKey, chunk, dialect));
}
/// <summary>
/// Emits <c>SELECT &lt;key&gt;, &lt;value&gt;[, &lt;ts&gt;] FROM &lt;table&gt; WHERE &lt;key&gt; IN (@k0..@kN)</c>
/// for one chunk of a group. The key column is selected because the reader indexes rows by it to slice
/// values back per member.
/// <para>Keys are de-duplicated (ordinal) before binding: two tags authored against the same
/// <c>keyValue</c> bind one parameter but stay two members, so both nodes are fed from the one row.</para>
/// <para>Markers are numbered from <c>k0</c> within the chunk — each chunk is its own command, so
/// carrying a group-wide index forward would leave a three-parameter command spelling
/// <c>IN (@k3, @k4, @k5)</c>.</para>
/// </summary>
private static SqlQueryPlan BuildKeyValuePlan(
object groupKey, List<SqlTagDefinition> members, ISqlDialect dialect)
@@ -27,6 +27,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql;
/// <param name="GroupKey">
/// The opaque equality key that decided this grouping — a value tuple, so two plans compare equal exactly
/// when they would have folded together. Exposed for diagnostics and plan caching, never for parsing.
/// <para><b>Not unique across plans.</b> A key-value group holding more distinct keys than the dialect's
/// <see cref="ISqlDialect.MaxBindParameters"/> is emitted as several plans carrying <em>this same key</em>
/// — the chunking is an execution detail of one group, not a regrouping.</para>
/// </param>
/// <param name="SqlText">The single parameterized statement to execute for this group.</param>
/// <param name="ParameterNames">