dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

merge: SQL driver health review findings resolved (Driver.Sql-004/-005/-006/-008/-010)

Browse Source 
# Conflicts:
#	docs/drivers/Sql.md
This commit is contained in:
Joseph Doherty
2026-08-10 08:11:40 -04:00
parent ead8a97511 bb83099300
commit b8acabfa5f
5 changed files with 497 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs
+179 -36
View File
@@ -155,6 +155,20 @@ public sealed class SqlDriver
private DriverHealth _health = new(DriverState.Unknown, null, null);
/// <summary>
/// The exception instance <see cref="ReadAsync"/> has already turned into a health verdict, held only
/// until <see cref="HandlePollError"/> — which the poll engine calls with that same object — has
/// consumed it (Driver.Sql-004).
/// </summary>
/// <remarks>
/// A field rather than a flag on the exception itself: <see cref="Exception.Data"/> is provider-owned
/// and may be read-only, and wrapping the throw would change the type the engine and
/// <see cref="IReadable"/>'s callers see. Concurrent poll loops can overwrite each other's mark; the
/// loser is then merely re-degraded with a poorer message, which is the pre-fix behaviour rather than
/// a new failure.
/// </remarks>
private Exception? _classifiedByRead;
/// <summary>Occurs when a subscribed tag's value or quality changes.</summary>
public event EventHandler<DataChangeEventArgs>? OnDataChange;
@@ -528,6 +542,10 @@ public sealed class SqlDriver
// on the log sink via the LogWarning exception parameter above.
Degrade($"Sql driver '{_driverInstanceId}' could not reach {Endpoint}: {ex.GetType().Name}.");
TransitionHostTo(HostState.Stopped);
// Mark THIS exception object as already classified before it goes back to the poll engine, which
// routes it to HandlePollError next. The mark is what makes the deferral there exact — see that
// method for why inferring it from the exception's type was not (Driver.Sql-004).
Volatile.Write(ref _classifiedByRead, ex);
throw;
}
}
@@ -577,11 +595,16 @@ public sealed class SqlDriver
/// <summary>
/// Classifies one poll's snapshots into a health verdict and a host state.
/// <list type="bullet">
/// <item>Any Good snapshot ⇒ the database answered: <c>LastSuccessfulRead</c> advances and the
/// host is Running.</item>
/// <item>Any <b>connection-class</b> Bad snapshot (<see cref="SqlStatusCodes.BadTimeout"/> /
/// <see cref="SqlStatusCodes.BadCommunicationError"/>) ⇒ Degraded; when nothing at all was Good,
/// the host is Stopped.</item>
/// <item>Any Good snapshot ⇒ Healthy and the host is Running.</item>
/// <item>Any snapshot that is <b>evidence the database answered</b>
/// (<see cref="IsAnswerEvidence"/> — Good, but also an absent row, a NULL cell, an uncoercible
/// cell) ⇒ <c>LastSuccessfulRead</c> advances and the host is Running, <em>without</em> changing
/// the health state. That clock is what the comms-loss watchdog reads (Driver.Sql-006).</item>
/// <item>Any <b>connection-class</b> Bad snapshot (<see cref="SqlStatusCodes.BadTimeout"/>) ⇒
/// Degraded; when nothing at all answered, the host is Stopped.</item>
/// <item>Any <b>query-failure</b> Bad snapshot (<see cref="SqlStatusCodes.BadCommunicationError"/>)
/// ⇒ Degraded, but the host stays Running. See <see cref="IsQueryFailureClass"/>: that code is
/// minted only after the connection opened, so it is evidence the endpoint is up.</item>
/// <item>Bad codes that are <b>authoring</b> facts — an unresolvable RawPath, an absent row, a
/// type mismatch, a rejected definition — change nothing. A tag typo must never report the
/// database as down and send an operator to the wrong system.</item>
@@ -602,18 +625,22 @@ public sealed class SqlDriver
var good = 0;
var unreachable = 0;
var queryFailed = 0;
var answered = 0;
foreach (var snapshot in snapshots)
{
if (SqlStatusCodes.IsGood(snapshot.StatusCode)) good++;
else if (IsConnectionClass(snapshot.StatusCode)) unreachable++;
else if (IsQueryFailureClass(snapshot.StatusCode)) queryFailed++;
if (IsAnswerEvidence(snapshot.StatusCode)) answered++;
}
if (unreachable > 0)
{
Degrade(
$"{unreachable} of {snapshots.Count} Sql tag(s) timed out or failed communication against " +
$"{Endpoint} on the last poll.");
if (good > 0)
$"{unreachable} of {snapshots.Count} Sql tag(s) timed out against {Endpoint} on the last poll.");
if (answered > 0)
{
// Partial: something answered, so the endpoint is up — but the driver is not healthy.
TouchLastSuccessfulRead();
@@ -627,19 +654,92 @@ public sealed class SqlDriver
return;
}
if (good == 0) return; // authoring-class Bad only — not a statement about the database.
if (queryFailed > 0)
{
// The connection opened and the server answered — with an error. Degraded (every tag in that
// group is Bad) but emphatically NOT a reachability verdict: reporting the endpoint down over a
// dropped table or a revoked SELECT points the operator at the wrong system, and the reconnect
// it would eventually provoke cannot fix a query.
Degrade(
$"{queryFailed} of {snapshots.Count} Sql tag(s) failed their query against {Endpoint} on the " +
"last poll; the connection opened, so check the table/column, the grant and any lock rather " +
"than the network.");
TouchLastSuccessfulRead();
TransitionHostTo(HostState.Running);
return;
}
if (answered == 0) return; // authoring-class Bad only — not a statement about the database.
if (good == 0)
{
// The database answered, just not with a value: an absent row, a NULL cell, a cell that would
// not coerce. Those are data/authoring facts, so the health STATE is deliberately left exactly
// as it was — but they are still proof the transport works, and the evidence clock the
// comms-loss watchdog reads must say so (Driver.Sql-006).
TouchLastSuccessfulRead();
TransitionHostTo(HostState.Running);
return;
}
// Route through the Faulted guard, not a raw WriteHealth: a poll still in flight when a concurrent
// ReinitializeAsync has recorded Faulted (the engine's ~5 s teardown wait is shorter than the 15 s
// default operationTimeout, so a stale poll can still complete afterwards) must not flip Faulted back
// to Healthy with no real recovery. Only a successful (re)initialize clears Faulted.
SetHealthUnlessFaulted(new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null));
RecordHealthyRead();
TransitionHostTo(HostState.Running);
}
/// <summary>True for the status codes that mean "the database did not answer", as opposed to "the data is not there".</summary>
private static bool IsConnectionClass(uint statusCode)
=> statusCode is SqlStatusCodes.BadTimeout or SqlStatusCodes.BadCommunicationError;
/// <summary>
/// True for the status codes that mean "the database did not answer", as opposed to "the data is not
/// there" or "the query was refused".
/// <para><see cref="SqlStatusCodes.BadCommunicationError"/> is deliberately <b>not</b> here despite
/// its name (Driver.Sql-005): the reader mints it from <c>GroupResult.Failed</c>, which is reachable
/// only after <c>OpenAsync</c> returned. Treating it as "did not answer" made one group failing on a
/// schema change, a lock or a revoked grant report the whole endpoint down. See
/// <see cref="IsQueryFailureClass"/>.</para>
/// </summary>
/// <param name="statusCode">The status code to classify.</param>
/// <returns>True when the code means the database did not answer at all.</returns>
private static bool IsConnectionClass(uint statusCode) => statusCode is SqlStatusCodes.BadTimeout;
/// <summary>
/// True for the status codes that mean "the connection opened but the query failed" — a schema
/// change, a lock, a missing grant, a statement the provider rejected. Degrading, but proof the
/// endpoint is up.
/// </summary>
/// <param name="statusCode">The status code to classify.</param>
/// <returns>True when the code means the server answered the connection and refused the query.</returns>
private static bool IsQueryFailureClass(uint statusCode)
=> statusCode is SqlStatusCodes.BadCommunicationError;
/// <summary>
/// True for the status codes that are proof the database <b>answered</b> this poll — the evidence
/// <c>LastSuccessfulRead</c> records, which is not the same question as "is this value good".
/// </summary>
/// <remarks>
/// <para>Driver.Sql-006. <c>DriverInstanceActor</c>'s comms-loss watchdog force-reconnects a driver
/// that is non-Healthy <b>and</b> whose <c>LastSuccessfulRead</c> has been frozen for 90 s (Gitea
/// #527), on the stated premise that the clock is the honest evidence the transport works. Advancing
/// it on Good alone made a truncated staging table or a deleted key set look — to that watchdog —
/// exactly like a dead socket, so a database answering every poll on time got its connection torn
/// down and rebuilt every 90 s, forever.</para>
/// <para>What is deliberately excluded is as load-bearing as what is included:
/// <see cref="SqlStatusCodes.BadTimeout"/> is the database <em>not</em> answering, and
/// <see cref="SqlStatusCodes.BadNodeIdUnknown"/> / <see cref="SqlStatusCodes.BadConfigurationError"/>
/// / <see cref="SqlStatusCodes.BadInternalError"/> are minted before any statement is executed. A
/// blanket "anything that is not a timeout" would keep the clock fresh on a driver that never once
/// reached the database, which disables the watchdog rather than fixing it.</para>
/// </remarks>
/// <param name="statusCode">The status code to classify.</param>
/// <returns>True when the code could only have been produced by a database that answered.</returns>
private static bool IsAnswerEvidence(uint statusCode) => statusCode is
SqlStatusCodes.Good // the row is there and readable
or SqlStatusCodes.BadNoData // the query ran; that row is absent
or SqlStatusCodes.Uncertain // a present row with a NULL cell (nullIsBad=false)
or SqlStatusCodes.Bad // …the same row under nullIsBad=true
or SqlStatusCodes.BadTypeMismatch // a cell came back and would not coerce
or SqlStatusCodes.BadCommunicationError; // the connection opened; the server refused the query
/// <summary>
/// Routes a poll-loop reader failure to the health surface (05/STAB-9). Deliberately does <b>not</b>
@@ -650,11 +750,20 @@ public sealed class SqlDriver
/// <param name="ex">The exception the poll engine caught.</param>
internal void HandlePollError(Exception ex)
{
// A subscribed-read outage surfaces as a DbException the reader threw, which ReadAsync has ALREADY
// classified — good, endpoint-bearing LastError + host Stopped — before rethrowing. The poll engine
// then routes that same exception here. Re-degrading would overwrite the good LastError with a worse,
// guidance-free one and log the outage a second time, so skip the connection class ReadAsync owns.
if (ex is DbException) return;
// A subscribed-read outage is classified by ReadAsync — good, endpoint-bearing LastError + host
// Stopped — before it rethrows, and the poll engine then routes that SAME exception object here.
// Re-degrading would overwrite the good LastError with a worse, guidance-free one and log the outage
// a second time, so defer to the classification that already happened.
//
// Matched on the exception INSTANCE ReadAsync marked, not on its type. `ex is DbException` was the
// stated intent — "the class ReadAsync owns" — but not what it implemented (Driver.Sql-004): the
// reader raises a plain InvalidOperationException when the provider factory returns no connection,
// and a provider can raise ArgumentException/FormatException off a malformed connection string on
// the same open path. Those are classified by ReadAsync exactly like a DbException, yet a type test
// let them through and clobbered its message; conversely a DbException that never went through
// ReadAsync — the poll engine's own throw — was swallowed. Consumed once, so a later unclassified
// exception is never mistaken for this one.
if (ReferenceEquals(Interlocked.CompareExchange(ref _classifiedByRead, null, ex), ex)) return;
// Whatever reaches here — an engine contract-violation, or a non-DbException the provider raised
// (e.g. an ArgumentException from a malformed connection string, which the parser echoes with
@@ -673,36 +782,70 @@ public sealed class SqlDriver
/// </summary>
/// <param name="reason">The operator-facing reason, which must never carry the connection string.</param>
private void Degrade(string reason)
{
var current = ReadHealth();
SetHealthUnlessFaulted(new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead, reason));
}
=> UpdateHealthUnlessFaulted(
current => new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead, reason));
/// <summary>
/// Publishes <paramref name="value"/> unless the driver is already <see cref="DriverState.Faulted"/>
/// — a liveness/config verdict that only a successful (re)initialize may clear. Every non-Initialize
/// health write routes through here: <see cref="Degrade"/> and the poll's Healthy verdict in
/// <see cref="ObservePollOutcome"/>, so a late in-flight poll cannot un-fault a driver a concurrent
/// <see cref="ReinitializeAsync"/> just faulted.
/// Records a poll that returned at least one Good value: Healthy, with the evidence clock advanced.
/// <para>The timestamp is taken <b>inside</b> the transform, so a retried compare-and-swap publishes
/// a fresh one rather than the one this call started with — which a concurrent advance could already
/// have overtaken.</para>
/// </summary>
/// <param name="value">The health snapshot to publish when the driver is not Faulted.</param>
private void SetHealthUnlessFaulted(DriverHealth value)
{
if (ReadHealth().State == DriverState.Faulted) return;
WriteHealth(value);
}
private void RecordHealthyRead()
=> UpdateHealthUnlessFaulted(
static _ => new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null));
/// <summary>Advances <c>LastSuccessfulRead</c> without changing the current state or error.</summary>
private void TouchLastSuccessfulRead()
=> UpdateHealth(static current => current with { LastSuccessfulRead = DateTime.UtcNow });
/// <summary>
/// Applies <paramref name="project"/> unless the driver is already <see cref="DriverState.Faulted"/>
/// — a liveness/config verdict that only a successful (re)initialize may clear. Every non-Initialize
/// state write routes through here: <see cref="Degrade"/> and the poll's Healthy verdict in
/// <see cref="ObservePollOutcome"/>, so a late in-flight poll cannot un-fault a driver a concurrent
/// <see cref="ReinitializeAsync"/> just faulted. The guard is evaluated against the <em>same</em>
/// snapshot the write then swaps out, so a Faulted verdict landing mid-update cannot be lost.
/// </summary>
/// <param name="project">Builds the snapshot to publish from the current one.</param>
private void UpdateHealthUnlessFaulted(Func<DriverHealth, DriverHealth> project)
=> UpdateHealth(current => current.State == DriverState.Faulted ? null : project(current));
/// <summary>
/// The one read-modify-write of the health field, as a compare-and-swap retry loop.
/// </summary>
/// <remarks>
/// Driver.Sql-010. Health is transitioned from every poll loop (one per subscription) plus any
/// ad-hoc <see cref="ReadAsync"/>, and every transition needs the current snapshot to build the next
/// one — <see cref="Degrade"/> preserves <c>LastSuccessfulRead</c>, the Faulted guard reads the state.
/// A plain <c>Volatile</c> read followed by a <c>Volatile</c> write lets a concurrent update land in
/// between and be silently overwritten: an advance of the evidence clock the comms-loss watchdog
/// reads is lost (and the clock even moves backwards), or a Faulted verdict is downgraded despite the
/// guard. <paramref name="transform"/> is re-run on each attempt, so it must be side-effect-free and
/// must derive everything — timestamps included — from the snapshot it is handed or from the clock.
/// Returning null declines the update.
/// </remarks>
/// <param name="transform">Builds the snapshot to publish, or returns null to leave health untouched.</param>
private void UpdateHealth(Func<DriverHealth, DriverHealth?> transform)
{
var current = ReadHealth();
WriteHealth(current with { LastSuccessfulRead = DateTime.UtcNow });
while (true)
{
var current = Volatile.Read(ref _health);
var next = transform(current);
if (next is null) return;
if (ReferenceEquals(Interlocked.CompareExchange(ref _health, next, current), current)) return;
}
}
/// <summary>Barrier-protected read of the multi-threaded health field.</summary>
private DriverHealth ReadHealth() => Volatile.Read(ref _health);
/// <summary>Barrier-protected publish of a new health snapshot.</summary>
/// <summary>
/// Unconditional publish of a new health snapshot, for the lifecycle transitions that are
/// last-writer-wins by definition (Initialize / Shutdown). Everything driven by a poll goes through
/// <see cref="UpdateHealth"/> instead, because it has to read the current snapshot first.
/// </summary>
/// <param name="value">The health snapshot to publish.</param>
private void WriteHealth(DriverHealth value) => Volatile.Write(ref _health, value);
/// <summary>Records a host-state change and raises <see cref="OnHostStatusChanged"/> — on transitions only.</summary>
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs
+12 -2
View File
@@ -566,9 +566,19 @@ public sealed class SqlPollReader
/// one. Worse, it would destroy the only evidence this method has that the selector is ambiguous
/// (<c>Rows.Count &gt; 1</c>), converting a loud misconfiguration into a permanently silent one —
/// precisely the failure mode this class's summary calls its top risk. The <c>topByTimestamp</c>
/// form keeps its row limit because there the <c>ORDER BY</c> makes "the first row" a defined
/// answer. The accepted cost is that an ambiguous selector materialises every matching row for one
/// form keeps its row limit because there the <c>ORDER BY</c> at least fixes <em>which</em> rows can
/// be first. The accepted cost is that an ambiguous selector materialises every matching row for one
/// poll; the warning is what gets it fixed.</para>
/// <para><b>⚠ The <c>topByTimestamp</c> answer is only defined when the ordering column is unique</b>
/// (Driver.Sql-008). <c>ORDER BY ts DESC</c> plus a one-row limit over a column with ties returns an
/// arbitrary row among the tied ones, and — unlike the where-pair form — this method can never
/// detect it, because the server discarded the other rows before the result set was built: there is
/// no <c>Rows.Count &gt; 1</c> left to warn about. The tie shape is reachable rather than
/// theoretical, since <c>SqlBrowseSession.PreferredTimestampColumn</c> will nominate a
/// <c>date</c>-typed column, which ties for every row written on the same day. The model's premise is
/// unchanged — the selector must identify one row — but for this form the operator, not the driver,
/// is the only thing that can check it. Point the tag at a column that is unique per row (or at a
/// latest-row view).</para>
/// </summary>
private object?[]? SelectWideRow(SqlQueryPlan plan, GroupResult outcome)
{