Merge feat/671-time-weighted-aggregates: Part 13 time-weighted aggregates (#671)

This commit is contained in:
Joseph Doherty
2026-10-11 09:10:52 -04:00
29 changed files with 1502 additions and 47 deletions
@@ -47,6 +47,12 @@ public enum HistoryAggregateType
AnnotationCount,
StandardDeviationPopulation,
VariancePopulation,
/// <summary>Part 13 TimeAverage2: time-weighted average over the interval's non-Bad data only (#671).</summary>
TimeAverage2,
/// <summary>Part 13 Total2: time-weighted integral (value·seconds) over the interval's non-Bad data only (#671).</summary>
Total2,
}
/// <summary>
@@ -65,6 +71,22 @@ public sealed class HistoryAggregateNotSupportedException : NotSupportedExceptio
=> Aggregate = aggregate;
}
/// <summary>
/// Thrown by a history backend when a read would exceed a configured size limit (e.g. the raw-row cap of the SQL
/// backends' time-weighted aggregates, #671). The Server HistoryRead path translates it to
/// <c>BadTooManyOperations</c> for that node only; it is a client-request limit, not a backend fault.
/// </summary>
public sealed class HistoryReadLimitExceededException : InvalidOperationException
{
/// <summary>The configured limit that was exceeded.</summary>
public long Limit { get; }
/// <summary>Creates the exception.</summary>
/// <param name="limit">The configured limit.</param>
/// <param name="message">Operator-facing description.</param>
public HistoryReadLimitExceededException(long limit, string message) : base(message) => Limit = limit;
}
/// <summary>
/// One row returned by <see cref="IHistorianDataSource.ReadEventsAsync"/> — a historical
/// alarm/event record, not the OPC UA live-event stream. Fields match the minimum set the
@@ -105,6 +105,42 @@ internal static class QuestDbAggregateQueries
"SELECT st.*, r.num_value, r.ts " +
"FROM st LEFT JOIN (SELECT * FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b";
/// <summary>
/// Time-weighted family (#671), bounds statement — <b>not</b> pushed down (see <see cref="TimeWeightedWindow"/>).
/// Four <c>LIMIT 1</c> seeks <c>UNION ALL</c>-ed, each tagged with <c>k</c>: 0 = latest usable sample before the
/// start, 1 = latest sample of any quality before the start, 2 = first sample of any quality at or after the end,
/// 3 = first usable one at or after the end. Parameters: <c>$1</c> tag, <c>$2</c>/<c>$3</c> the forward window
/// start / end. Columns: <c>k</c>, then <c>ts, kind, num_value, int_value, str_value, status, server_ts</c>.
/// </summary>
/// <param name="sample">Prefixed sample table name.</param>
/// <returns>The SQL text.</returns>
public static string TimeWeightedBounds(string sample)
{
var usable = $" AND status < {NotBadLimit} AND kind <> {EmptyKind}";
string Seek(int k, string cmp, string filter, string order) =>
$"SELECT * FROM (SELECT {k.ToString(CultureInfo.InvariantCulture)} AS k, {SampleColumns} FROM {sample} " +
$"WHERE tag = $1 AND ts {cmp}{filter} AND NOT deleted ORDER BY ts {order} LIMIT 1)";
return Seek(0, "< $2", usable, "DESC") + " UNION ALL " + Seek(1, "< $2", "", "DESC") + " UNION ALL " +
Seek(2, ">= $3", "", "ASC") + " UNION ALL " + Seek(3, ">= $3", usable, "ASC");
}
/// <summary>
/// Time-weighted family (#671), window statement: the live samples in <c>[$2, $3)</c> for tag <c>$1</c>, ascending,
/// at most <c>$4</c> (the raw-row cap + 1).
/// Columns: <c>ts, kind, num_value, int_value, str_value, status, server_ts</c>. The caller feeds the before-bounds,
/// these rows and the after-bounds, in that order, into <see cref="TimeWeightedAggregator"/> — the same evaluator
/// the SQL Server backend uses, so the two agree exactly. Trade-off: the window's rows travel to the server rather
/// than one row per bucket; Part 13's segment interpolation and bucket-edge splitting are then written once
/// instead of twice in two SQL dialects that must stay bit-identical.
/// </summary>
/// <param name="sample">Prefixed sample table name.</param>
/// <returns>The SQL text.</returns>
public static string TimeWeightedWindow(string sample) =>
$"SELECT {SampleColumns} FROM {sample} WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts ASC LIMIT $4";
/// <summary>The row columns the time-weighted statements return (after <c>k</c> for the bounds).</summary>
private const string SampleColumns = "ts, kind, num_value, int_value, str_value, status, server_ts";
/// <summary>The bucket-index expression (exposed for tests).</summary>
internal const string BucketExpression = "(cast(ts AS LONG) * 10 - $1) / $2";
@@ -64,6 +64,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
private readonly ILogger _log;
private readonly TimeProvider _time;
private readonly HistorianHealthCounters _health = new();
private readonly int _maxTimeWeightedRawRows;
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
/// <param name="conn">Connection helper.</param>
@@ -71,10 +72,12 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
/// <param name="catalog">Tag catalog (reads use only its read-only <see cref="QuestDbTagCatalog.ExistsAsync"/>).</param>
/// <param name="log">Optional logger.</param>
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (&lt;= 0 = unbounded).</param>
internal QuestDbHistorianDataSource(
QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbTagCatalog catalog, ILogger? log = null,
TimeProvider? timeProvider = null)
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
{
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
@@ -332,9 +335,10 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
/// the <see cref="BucketRow"/> statistics of each non-empty bucket (bucket index
/// <c>(cast(ts AS LONG)*10 − StartUnixTicks) / IntervalTicks</c>, never <c>SAMPLE BY</c>);
/// <see cref="AggregateEvaluator.Evaluate"/> produces each value and <see cref="BucketPlan.Assemble"/> fills the
/// empty buckets with <c>BadNoData</c>.
/// empty buckets with <c>BadNoData</c>. The time-weighted family (#671) instead streams the window plus four bounding
/// rows into <see cref="TimeWeightedAggregator"/>, exactly as the SQL Server backend does.
/// </remarks>
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the thirteen served.</exception>
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the seventeen served.</exception>
/// <exception cref="ArgumentOutOfRangeException">The window holds more than <see cref="BucketPlan.MaxBuckets"/> intervals.</exception>
public async Task<HistoryReadResult> ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
@@ -348,6 +352,15 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
{
var now = _time.GetUtcNow().UtcDateTime;
var evaluated = new Dictionary<long, DataValueSnapshot>();
if (family == AggregateFamily.TimeWeighted)
{
var tw = new TimeWeightedAggregator(aggregate, plan);
if (plan.BucketCount > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
await StreamTimeWeightedAsync(fullReference, plan, tw, cancellationToken).ConfigureAwait(false);
var twResult = tw.Finish(now);
RecordSuccess();
return new HistoryReadResult(twResult, ContinuationPoint: null);
}
if (plan.BucketCount > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
{
var sample = _conn.SampleTable;
@@ -377,6 +390,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
RecordSuccess();
return new HistoryReadResult(result, ContinuationPoint: null);
}
catch (HistoryReadLimitExceededException)
{
// A client-request limit (the raw-row cap), not a backend fault: not counted as a failure.
throw;
}
catch (Exception ex)
{
RecordFailure(ex);
@@ -384,6 +402,47 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
}
}
/// <summary>
/// Streams the time-weighted bounds (<see cref="QuestDbAggregateQueries.TimeWeightedBounds"/>) and window
/// (<see cref="QuestDbAggregateQueries.TimeWeightedWindow"/>) into <paramref name="tw"/> in time order: the two
/// before-bounds, the window, the two after-bounds (duplicates are dropped by the aggregator).
/// </summary>
private async Task StreamTimeWeightedAsync(string tag, BucketPlan plan, TimeWeightedAggregator tw, CancellationToken ct)
{
var bounds = new StoredSample?[4];
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
await using (var cmd = _conn.CreateCommand(conn, QuestDbAggregateQueries.TimeWeightedBounds(_conn.SampleTable)))
{
QuestDbConnection.BindText(cmd, tag);
QuestDbConnection.BindTimestamp(cmd, plan.StartUtc);
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
bounds[Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture)] = ReadStored(reader, 1);
}
if (bounds[0] is { } usableBefore) tw.Add(usableBefore);
if (bounds[1] is { } anyBefore) tw.Add(anyBefore);
await using (var cmd = _conn.CreateCommand(conn, QuestDbAggregateQueries.TimeWeightedWindow(_conn.SampleTable)))
{
QuestDbConnection.BindText(cmd, tag);
QuestDbConnection.BindTimestamp(cmd, plan.StartUtc);
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
cmd.Parameters.Add(new NpgsqlParameter
{ NpgsqlDbType = NpgsqlDbType.Integer, Value = TimeWeightedAggregator.WindowFetchLimit(_maxTimeWeightedRawRows) });
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
long windowRows = 0;
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
// LIMIT $4 = cap + 1: one row past the cap fails the node without reading on.
TimeWeightedAggregator.CheckWindowRows(++windowRows, _maxTimeWeightedRawRows, tag);
tw.Add(ReadStored(reader, 0));
}
}
if (bounds[2] is { } anyAfter) tw.Add(anyAfter);
if (bounds[3] is { } usableAfter) tw.Add(usableAfter);
}
private static BucketRow ReadBucket(NpgsqlDataReader reader, AggregateFamily family)
{
static double? D(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
@@ -1,3 +1,4 @@
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
using System.Collections.Frozen;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
@@ -33,11 +34,16 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
/// <summary>Creates the provider over the <c>ServerHistorian:QuestDb</c> options (validated elsewhere).</summary>
/// <param name="options">The <c>ServerHistorian:QuestDb</c> options.</param>
public QuestDbHistorianProvider(QuestDbHistorianOptions options)
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (&lt;= 0 = unbounded).</param>
public QuestDbHistorianProvider(QuestDbHistorianOptions options,
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
}
private readonly int _maxTimeWeightedRawRows;
/// <inheritdoc />
public string Name => "QuestDb";
@@ -69,7 +75,7 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
{
var s = Get(sp);
return new QuestDbHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>());
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
}
/// <inheritdoc />
@@ -17,7 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// <para>
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
/// <see cref="double"/> computed over the numeric value (Boolean = 0/1). These are the simple per-sample
/// definitions, not Part 13's interval-boundary (interpolated bound / time-weighted) semantics.
/// definitions; the Part 13 time-weighted aggregates (TimeAverage, TimeAverage2, Total, Total2, Interpolative)
/// are evaluated by <see cref="TimeWeightedAggregator"/>, not here.
/// </para>
/// </remarks>
public static class AggregateEvaluator
@@ -28,7 +29,8 @@ public static class AggregateEvaluator
/// <param name="bucketStart">The bucket's start time (the result timestamp).</param>
/// <param name="now">Server timestamp.</param>
/// <returns>The calculated value.</returns>
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the thirteen served.</exception>
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the per-sample (Simple /
/// StartEnd / ActualTime family) aggregates.</exception>
public static DataValueSnapshot Evaluate(HistoryAggregateType aggregate, BucketRow r, DateTime bucketStart, DateTime now)
{
DataValueSnapshot Calc(object? value, uint status, DateTime? t = null) =>
@@ -53,7 +55,6 @@ public static class AggregateEvaluator
case HistoryAggregateType.Average: return Calc(r.Avg, status);
case HistoryAggregateType.Minimum: return Calc(r.Min, status);
case HistoryAggregateType.Maximum: return Calc(r.Max, status);
case HistoryAggregateType.Total: return Calc(r.Sum, status);
case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status);
case HistoryAggregateType.Start: return Calc(r.First.ToObject(), status);
case HistoryAggregateType.End: return Calc(r.Last.ToObject(), status);
@@ -5,7 +5,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// <summary>The query shape that serves an aggregate. One query per family.</summary>
public enum AggregateFamily
{
/// <summary>Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad: one GROUP BY.</summary>
/// <summary>Average / Minimum / Maximum / Count / Range / PercentGood / PercentBad: one GROUP BY.</summary>
Simple,
/// <summary>Start / End / Delta: first + last usable sample per bucket.</summary>
@@ -13,6 +13,13 @@ public enum AggregateFamily
/// <summary>MinimumActualTime / MaximumActualTime: extreme usable sample per bucket, ties ⇒ earliest.</summary>
ActualTime,
/// <summary>
/// TimeAverage / TimeAverage2 / Total / Total2 / Interpolative: Part 13 time-weighted, with bounding values.
/// The backend streams the window's samples plus the bounding rows and <see cref="TimeWeightedAggregator"/>
/// evaluates them (identically for every backend).
/// </summary>
TimeWeighted,
}
/// <summary>Which aggregates the SQL historian backends serve, and by which family.</summary>
@@ -24,10 +31,12 @@ public static class AggregateFamilies
public static AggregateFamily? FamilyOf(HistoryAggregateType aggregate) => aggregate switch
{
HistoryAggregateType.Average or HistoryAggregateType.Minimum or HistoryAggregateType.Maximum
or HistoryAggregateType.Total or HistoryAggregateType.Count or HistoryAggregateType.Range
or HistoryAggregateType.Count or HistoryAggregateType.Range
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => AggregateFamily.Simple,
HistoryAggregateType.Start or HistoryAggregateType.End or HistoryAggregateType.Delta => AggregateFamily.StartEnd,
HistoryAggregateType.MinimumActualTime or HistoryAggregateType.MaximumActualTime => AggregateFamily.ActualTime,
HistoryAggregateType.TimeAverage or HistoryAggregateType.TimeAverage2 or HistoryAggregateType.Total
or HistoryAggregateType.Total2 or HistoryAggregateType.Interpolative => AggregateFamily.TimeWeighted,
_ => null,
};
}
@@ -12,6 +12,12 @@ public static class HistorianInfoBits
/// <summary>The value was interpolated.</summary>
public const uint HistorianInterpolated = 0x2u;
/// <summary>
/// Partial (Part 4 §7.39 / <c>Opc.Ua.AggregateBits.Partial</c>): the calculated value covers an interval shorter
/// than the processing interval (the truncated last interval of a processed read).
/// </summary>
public const uint HistorianPartial = 0x4u;
/// <summary>
/// Marks <paramref name="status"/> as a Calculated data value. Any historian bits the status already carries
/// (e.g. a stored Interpolated sample) are cleared first, so the result is never both.
@@ -10,6 +10,9 @@ public static class HistorianStatusCodes
public const uint GoodEntryInserted = 0x00A20000u;
public const uint GoodEntryReplaced = 0x00A30000u;
public const uint BadNoData = 0x809B0000u;
/// <summary>Generic Bad: a time-weighted interval holding data but no non-Bad time (the SDK aggregate rule).</summary>
public const uint Bad = 0x80000000u;
public const uint BadEntryExists = 0x809F0000u;
public const uint BadNoEntryExists = 0x80A00000u;
public const uint BadInvalidTimestamp = 0x80230000u;
@@ -0,0 +1,430 @@
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// <summary>
/// The OPC UA Part 13 time-weighted aggregates (<see cref="AggregateFamily.TimeWeighted"/>): <c>TimeAverage</c>,
/// <c>TimeAverage2</c>, <c>Total</c>, <c>Total2</c> and <c>Interpolative</c>, evaluated over a time-ordered stream of
/// stored samples. Both SQL backends fetch the same rows and feed them here, so the two produce identical results
/// by construction.
/// </summary>
/// <remarks>
/// <para>
/// <b>Input.</b> Samples in strictly ascending time: the bounding rows before the window (the latest
/// <i>usable</i> sample before <c>start</c>, then the latest sample of any quality before <c>start</c> — they differ
/// when an outage straddles the start), every in-window sample, then the bounding rows at/after the window end
/// (the first sample of any quality at or after <c>end</c>, then the first <i>usable</i> one). <see cref="Add"/>
/// ignores a sample at or before the last one, so a bound fetched twice is harmless. Memory is O(buckets), not O(samples).
/// </para>
/// <para>
/// <b>The signal (server defaults: TreatUncertainAsBad = false, PercentDataBad = PercentDataGood = 100,
/// UseSlopedExtrapolation = false; a client <c>AggregateConfiguration</c> is not honoured).</b> Between a usable
/// sample <c>p</c> and the next sample <c>s</c>:
/// <list type="bullet">
/// <item><c>s</c> usable, both Double/Int64 ⇒ <b>sloped</b> (linear <c>p → s</c>); quality = worse of the two.</item>
/// <item><c>s</c> usable, either Boolean ⇒ <b>stepped</b> (<c>p</c> held); quality = <c>p</c>'s.</item>
/// <item><c>s</c> Bad ⇒ <c>p</c> held up to the outage (never interpolated into it, as AtTime does); Uncertain.</item>
/// <item>no later sample at all ⇒ <c>p</c> held to the window end (stepped extrapolation); Uncertain.</item>
/// </list>
/// From a Bad sample (incl. an <c>Empty</c> outage marker) to the next sample, and before the first sample, the
/// signal is Bad (no data). This is the <i>simple bounding</i> signal.
/// </para>
/// <para>
/// The <i>interpolated bounding</i> signal (Part 13 "interpolated bounding values") skips Bad samples: it joins
/// consecutive <b>usable</b> samples the same way (sloped / stepped) straight across any outage, and holds the
/// last usable value to the window end. It is undefined before the first usable sample.
/// </para>
/// <list type="table">
/// <item><term>TimeAverage</term><description>∫ interpolated-bounding signal ÷ its defined duration in the interval.</description></item>
/// <item><term>Total</term><description><c>TimeAverage × interval seconds</c> (value·s) — Part 13 §5.4.3.8.</description></item>
/// <item><term>TimeAverage2</term><description>∫ simple-bounding signal over its non-Bad parts ÷ non-Bad duration (Bad intervals excluded).</description></item>
/// <item><term>Total2</term><description>∫ simple-bounding signal over its non-Bad parts, in value·s (= TimeAverage2 × non-Bad seconds).</description></item>
/// <item><term>Interpolative</term><description>The interpolated-bounding signal's value at each interval start —
/// for a reverse read, at <c>End − i·interval</c> (Part 13 anchors a reverse read at its later time), emitted
/// latest first.</description></item>
/// </list>
/// <para>
/// <b>Status (time-weighted four)</b> — the SDK <c>AggregateCalculator.GetTimeBasedStatusCode</c> rule with
/// TreatUncertainAsBad = false, so Uncertain time (Uncertain samples, a value held up to an outage, stepped
/// extrapolation past the last sample) counts as Good time: every instant of the interval non-Bad ⇒
/// <c>Good</c>; no non-Bad time but some Bad (outage) time ⇒ <c>Bad</c>; no data at all ⇒ <c>BadNoData</c>;
/// otherwise (PercentDataGood = PercentDataBad = 100) <c>UncertainDataSubNormal</c>. Time before the first sample
/// counts as Bad (no data) time. A usable String/DateTime sample contributing to the interval ⇒
/// <c>BadAggregateInvalidInputs</c>. Every status carries the Calculated bit, plus the Partial bit on a non-Bad
/// result of a truncated last interval; the timestamp is the interval start; the value is a
/// <see cref="double"/> (Boolean = 0/1, Int64 as double).
/// </para>
/// <para>
/// <b>Status (Interpolative).</b> A usable sample exactly at the point ⇒ its value with its own status
/// (raw, no historian bit). Otherwise no usable sample at or before ⇒ <c>BadNoData</c>; else the interpolated
/// value with the Interpolated bit and the status of the simple-bounding signal at that instant: <c>Good</c> on
/// a Good segment, <c>UncertainDataSubNormal</c> on an Uncertain one, inside an outage (interpolated across it)
/// or past the last sample (extrapolated).
/// </para>
/// </remarks>
public sealed class TimeWeightedAggregator
{
private const double TicksPerSecond = TimeSpan.TicksPerSecond;
/// <summary>
/// Default cap on the raw in-window rows one time-weighted read may stream (<c>ServerHistorian:MaxTimeWeightedRawRows</c>).
/// </summary>
public const int DefaultMaxWindowRows = 1_000_000;
/// <summary>
/// The <c>TOP</c>/<c>LIMIT</c> a backend puts on its window query for a configured cap: <c>cap + 1</c> (so one
/// row past the cap proves it was exceeded without reading on), or <see cref="int.MaxValue"/> when the cap is
/// &lt;= 0 (unbounded).
/// </summary>
/// <param name="maxWindowRows">The configured cap.</param>
/// <returns>The row limit to put on the window query.</returns>
public static int WindowFetchLimit(int maxWindowRows) =>
maxWindowRows <= 0 || maxWindowRows == int.MaxValue ? int.MaxValue : maxWindowRows + 1;
/// <summary>Throws when <paramref name="rowsRead"/> has passed a positive <paramref name="maxWindowRows"/>.</summary>
/// <param name="rowsRead">Window rows read so far.</param>
/// <param name="maxWindowRows">The configured cap (&lt;= 0 = unbounded).</param>
/// <param name="tag">The tag, for the message.</param>
/// <exception cref="HistoryReadLimitExceededException">The cap is exceeded.</exception>
public static void CheckWindowRows(long rowsRead, int maxWindowRows, string tag)
{
if (maxWindowRows > 0 && rowsRead > maxWindowRows)
throw new HistoryReadLimitExceededException(maxWindowRows,
$"The time-weighted aggregate read of '{tag}' covers more than {maxWindowRows} raw samples " +
"(ServerHistorian:MaxTimeWeightedRawRows); narrow the window or use a pushed-down aggregate such as Average.");
}
private readonly HistoryAggregateType _aggregate;
private readonly BucketPlan _plan;
private readonly long _start;
private readonly long _end;
// Simple-bounding (raw) signal, per bucket: ∫ over non-Bad time, non-Bad ticks, ticks inside an outage.
private readonly double[] _rawArea;
private readonly long[] _usableTicks;
private readonly long[] _badTicks;
// Interpolated-bounding (bridged) signal, per bucket.
private readonly double[] _bridgedArea;
private readonly long[] _bridgedTicks;
private readonly bool[] _invalid;
// Interpolative, per point (index = ascending time; see PointTicks).
private readonly DataValueSnapshot?[] _points;
private StoredSample? _prevAny;
private StoredSample? _prevUsable;
private long _nextPoint; // next point (ascending time) not yet classified
private readonly List<PendingPoint> _pending = []; // points awaiting the next usable sample
private bool _finished;
/// <summary>
/// What the simple-bounding signal is at a pending bucket start: no sample yet; on a segment from a usable sample
/// whose end is not known yet; or degraded (inside an outage, or held up to one).
/// </summary>
private enum PointQuality { NoPrior, OnSegment, Degraded }
private readonly record struct PendingPoint(long Bucket, long Ticks, PointQuality Quality);
/// <summary>Starts an evaluation of <paramref name="aggregate"/> over <paramref name="plan"/>.</summary>
/// <param name="aggregate">One of the <see cref="AggregateFamily.TimeWeighted"/> aggregates.</param>
/// <param name="plan">The bucket plan (forward window).</param>
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not time-weighted.</exception>
public TimeWeightedAggregator(HistoryAggregateType aggregate, BucketPlan plan)
{
ArgumentNullException.ThrowIfNull(plan);
if (AggregateFamilies.FamilyOf(aggregate) != AggregateFamily.TimeWeighted)
throw new HistoryAggregateNotSupportedException(aggregate);
_aggregate = aggregate;
_plan = plan;
_start = plan.StartUtc.Ticks;
_end = plan.EndUtc.Ticks;
var n = (int)plan.BucketCount;
_rawArea = new double[n];
_usableTicks = new long[n];
_badTicks = new long[n];
_bridgedArea = new double[n];
_bridgedTicks = new long[n];
_invalid = new bool[n];
_points = new DataValueSnapshot?[n];
}
/// <summary>Evaluates <paramref name="aggregate"/> over an already time-ordered sample sequence.</summary>
/// <param name="aggregate">One of the <see cref="AggregateFamily.TimeWeighted"/> aggregates.</param>
/// <param name="plan">The bucket plan.</param>
/// <param name="samples">Bounds + window samples, ascending by time.</param>
/// <param name="now">Server timestamp for the results.</param>
/// <returns>One value per bucket, in the plan's order (descending for a reverse window).</returns>
public static IReadOnlyList<DataValueSnapshot> Evaluate(
HistoryAggregateType aggregate, BucketPlan plan, IEnumerable<StoredSample> samples, DateTime now)
{
ArgumentNullException.ThrowIfNull(samples);
var acc = new TimeWeightedAggregator(aggregate, plan);
foreach (var s in samples) acc.Add(s);
return acc.Finish(now);
}
/// <summary>Feeds the next sample (ascending time; a sample at or before the previous one is ignored).</summary>
/// <param name="s">The sample.</param>
public void Add(StoredSample s)
{
if (_finished) throw new InvalidOperationException("The aggregation is already finished.");
var t = s.TimeUtc.Ticks;
if (_prevAny is { } last && t <= last.TimeUtc.Ticks) return;
// Interpolative: classify every bucket start before this sample by the simple-bounding signal there.
ClassifyPointsBefore(t);
// Simple-bounding segment [prevAny, s).
if (_prevAny is { IsUsable: true } p)
{
// Toward a usable sample: sloped or stepped; toward a Bad one: held up to the outage.
var sloped = s.IsUsable && IsSloped(p, s);
AddRaw(p, t, sloped ? s.Value.Number : p.Value.Number);
}
else if (_prevAny is { } bad)
{
AddBad(bad, t);
}
if (!s.IsUsable) DegradePending(); // the segment the pending points sit on runs into an outage
if (s.IsUsable)
{
if (_prevUsable is { } pu)
{
var sloped = IsSloped(pu, s);
AddBridged(pu, t, sloped ? s.Value.Number : pu.Value.Number);
}
ResolvePending(_prevUsable, s);
_prevUsable = s;
}
_prevAny = s;
}
/// <summary>Completes the evaluation (extrapolating past the last sample) and assembles the per-bucket results.</summary>
/// <param name="now">Server timestamp for the results.</param>
/// <returns>One value per bucket, in the plan's order (descending for a reverse window).</returns>
public IReadOnlyList<DataValueSnapshot> Finish(DateTime now)
{
if (!_finished)
{
_finished = true;
ClassifyPointsBefore(long.MaxValue);
if (_prevAny is { IsUsable: true } p && p.TimeUtc.Ticks < _end)
AddRaw(p, _end, p.Value.Number); // stepped extrapolation
else if (_prevAny is { } bad && bad.TimeUtc.Ticks < _end)
AddBad(bad, _end); // the outage runs to the window end
if (_prevUsable is { } pu && pu.TimeUtc.Ticks < _end)
AddBridged(pu, _end, pu.Value.Number);
ResolvePending(_prevUsable, next: null);
}
var byBucket = new Dictionary<long, DataValueSnapshot>((int)_plan.BucketCount);
for (long i = 0; i < _plan.BucketCount; i++)
byBucket[i] = Result(i, now);
return _plan.Assemble(byBucket, t => AggregateEvaluator.NoData(t, now));
}
// ------------------------------------------------------------------------------------------ Results
private DataValueSnapshot Result(long i, DateTime now)
{
if (_aggregate == HistoryAggregateType.Interpolative)
return _points[i] is { } point
? point with { ServerTimestampUtc = now }
: new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, PointTime(i), now);
var bucketStart = _plan.BucketStart(i);
var duration = BucketEnd(i) - BucketStartTicks(i);
var partial = duration < _plan.IntervalTicks; // only the truncated last interval
DataValueSnapshot Calc(object? value, uint status)
{
var code = HistorianInfoBits.WithCalculated(status);
if (partial && !StatusBits.IsBad(status)) code |= HistorianInfoBits.HistorianPartial;
return new DataValueSnapshot(value, code, bucketStart, now);
}
if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
if (_usableTicks[i] == 0)
return _badTicks[i] > 0 ? Calc(null, HistorianStatusCodes.Bad) : AggregateEvaluator.NoData(bucketStart, now);
var status = _usableTicks[i] == duration ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
return _aggregate switch
{
HistoryAggregateType.TimeAverage => Calc(_bridgedArea[i] / _bridgedTicks[i], status),
HistoryAggregateType.Total => Calc(_bridgedArea[i] / _bridgedTicks[i] * (duration / TicksPerSecond), status),
HistoryAggregateType.TimeAverage2 => Calc(_rawArea[i] / _usableTicks[i], status),
HistoryAggregateType.Total2 => Calc(_rawArea[i] / TicksPerSecond, status),
_ => throw new HistoryAggregateNotSupportedException(_aggregate),
};
}
// ------------------------------------------------------------------------------------------ Segments
private static bool IsNumeric(StoredSample s) =>
s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64 or HistorianValueKind.Boolean;
/// <summary>Sloped between two usable samples only when both are Double/Int64; anything Boolean is stepped.</summary>
private static bool IsSloped(StoredSample a, StoredSample b) =>
a.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
&& b.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64;
private long BucketStartTicks(long i) => _start + i * _plan.IntervalTicks;
private long BucketEnd(long i) => Math.Min(BucketStartTicks(i) + _plan.IntervalTicks, _end);
private void AddRaw(StoredSample from, long toTicks, double toValue) =>
Distribute(from, toTicks, toValue, (i, area, ticks) =>
{
_rawArea[i] += area;
_usableTicks[i] += ticks;
});
/// <summary>Counts <c>[from.t, toTicks)</c> as outage (Bad) time; no value, never marks a bucket invalid.</summary>
private void AddBad(StoredSample from, long toTicks)
{
var lo = Math.Max(from.TimeUtc.Ticks, _start);
var hi = Math.Min(toTicks, _end);
if (lo >= hi || _plan.BucketCount == 0) return;
for (var i = (lo - _start) / _plan.IntervalTicks; i < _plan.BucketCount; i++)
{
var bs = BucketStartTicks(i);
if (bs >= hi) break;
var a = Math.Max(lo, bs);
var b = Math.Min(hi, BucketEnd(i));
if (a < b) _badTicks[i] += b - a;
}
}
private void AddBridged(StoredSample from, long toTicks, double toValue) =>
Distribute(from, toTicks, toValue, (i, area, ticks) =>
{
_bridgedArea[i] += area;
_bridgedTicks[i] += ticks;
});
/// <summary>
/// Splits the linear segment <c>[from.t, toTicks)</c> (value <c>from.v → toValue</c>) at the bucket edges, clipped to
/// the window, and hands each bucket its trapezoid area (value·ticks) and duration. A non-numeric start sample
/// marks every bucket it touches invalid instead.
/// </summary>
private void Distribute(StoredSample from, long toTicks, double toValue, Action<long, double, long> add)
{
var t0 = from.TimeUtc.Ticks;
var lo = Math.Max(t0, _start);
var hi = Math.Min(toTicks, _end);
if (lo >= hi || _plan.BucketCount == 0) return;
var numeric = IsNumeric(from);
var v0 = from.Value.Number;
var span = (double)(toTicks - t0);
double ValueAt(long x) => toValue == v0 ? v0 : v0 + (toValue - v0) * ((x - t0) / span);
for (var i = (lo - _start) / _plan.IntervalTicks; i < _plan.BucketCount; i++)
{
var bs = BucketStartTicks(i);
if (bs >= hi) break;
var a = Math.Max(lo, bs);
var b = Math.Min(hi, BucketEnd(i));
if (a >= b) continue;
if (!numeric) { _invalid[i] = true; continue; }
add(i, (ValueAt(a) + ValueAt(b)) / 2d * (b - a), b - a);
}
}
// ------------------------------------------------------------------------------------------ Interpolative
/// <summary>
/// The time of Interpolative point <paramref name="j"/> (ascending time). Forward: the bucket starts
/// <c>start + j·interval</c>. Reverse (Part 13 anchors a reverse read at its later time, the request's start):
/// <c>End − (n−1−j)·interval</c>, i.e. <c>End, End − interval, …</c> once <see cref="BucketPlan.Assemble"/> has
/// reversed the order — so a reverse read 10:00 → 09:00 at 1 h yields the value at 10:00.
/// </summary>
private long PointTicks(long j) =>
_plan.Reversed ? _end - (_plan.BucketCount - 1 - j) * _plan.IntervalTicks : BucketStartTicks(j);
private DateTime PointTime(long j) => new(PointTicks(j), DateTimeKind.Utc);
/// <summary>Classifies every unclassified point strictly before <paramref name="beforeTicks"/>.</summary>
private void ClassifyPointsBefore(long beforeTicks)
{
if (_aggregate != HistoryAggregateType.Interpolative) return;
for (; _nextPoint < _plan.BucketCount; _nextPoint++)
{
var a = PointTicks(_nextPoint);
if (a >= beforeTicks) break;
if (_prevAny is not { } p)
{
_pending.Add(new PendingPoint(_nextPoint, a, PointQuality.NoPrior));
}
else if (p.IsUsable && p.TimeUtc.Ticks == a)
{
// A usable sample exactly at the interval start: its own value and status.
_points[_nextPoint] = IsNumeric(p)
? new DataValueSnapshot(p.Value.Number, p.Status, PointTime(_nextPoint), default)
: InvalidPoint(_nextPoint);
}
else if (!p.IsUsable)
{
_pending.Add(new PendingPoint(_nextPoint, a, PointQuality.Degraded));
}
else
{
// Inside a segment that starts at a usable sample; its quality is only known once the next sample
// arrives (beforeTicks is that sample's time, or MaxValue at Finish ⇒ extrapolated).
_pending.Add(new PendingPoint(_nextPoint, a, PointQuality.OnSegment));
}
}
}
/// <summary>Resolves the pending bucket starts against the bridging pair <c>(p, next)</c>.</summary>
private void ResolvePending(StoredSample? p, StoredSample? next)
{
if (_pending.Count == 0) return;
foreach (var pt in _pending)
{
var bucketStart = PointTime(pt.Bucket);
if (p is not { } prev || pt.Quality == PointQuality.NoPrior)
{
_points[pt.Bucket] = new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, bucketStart, default);
continue;
}
if (!IsNumeric(prev) || next is { } nn && !IsNumeric(nn))
{
_points[pt.Bucket] = InvalidPoint(pt.Bucket);
continue;
}
// Good only on a segment that runs straight from prev to next (no outage skipped, not extrapolated), and only
// when the samples that define the value are Good: both ends when sloped, prev alone when stepped.
double value;
bool good;
if (next is { } n && IsSloped(prev, n))
{
value = prev.Value.Number + (n.Value.Number - prev.Value.Number)
* ((double)(pt.Ticks - prev.TimeUtc.Ticks) / (n.TimeUtc.Ticks - prev.TimeUtc.Ticks));
good = pt.Quality == PointQuality.OnSegment && prev.IsGood && n.IsGood;
}
else
{
value = prev.Value.Number;
good = pt.Quality == PointQuality.OnSegment && next is not null && prev.IsGood;
}
var status = good ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
_points[pt.Bucket] = new DataValueSnapshot(value, HistorianInfoBits.WithInterpolated(status), bucketStart, default);
}
_pending.Clear();
}
/// <summary>Marks every pending on-segment point degraded: its segment ends at a Bad sample.</summary>
private void DegradePending()
{
for (var k = 0; k < _pending.Count; k++)
if (_pending[k].Quality == PointQuality.OnSegment)
_pending[k] = _pending[k] with { Quality = PointQuality.Degraded };
}
private DataValueSnapshot InvalidPoint(long bucket) =>
new(null, HistorianStatusCodes.BadAggregateInvalidInputs, PointTime(bucket), default);
}
@@ -91,6 +91,36 @@ internal static class SqlAggregateQueries
"SELECT st.*, r.NumValue, r.SourceTimeUtc " +
"FROM st LEFT JOIN r ON r.b = st.b AND r.rn = 1 ORDER BY st.b;";
/// <summary>
/// Time-weighted family (#671: TimeAverage / TimeAverage2 / Total / Total2 / Interpolative) — <b>not</b> pushed down.
/// One batch of five result sets, each <c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode</c>:
/// (1) the latest usable sample before <c>@start</c>, (2) the latest sample of any quality before <c>@start</c>,
/// (3) the samples in <c>[@start, @end)</c> ascending, at most <c>@maxRows</c> (the raw-row cap + 1), (4) the first sample of any quality at or after
/// <c>@end</c>, (5) the first usable one at or after <c>@end</c>. Each bound is a <c>TOP 1</c> seek on the
/// <c>(TagId, SourceTimeUtc)</c> key. The caller streams them, in that order, into
/// <see cref="TimeWeightedAggregator"/> (shared with QuestDB, so both backends agree exactly).
/// </summary>
/// <remarks>
/// Trade-off: the window's rows travel to the server instead of one row per bucket. Part 13's per-segment
/// interpolation, bucket-edge splitting and the two bounding-value definitions would otherwise have to be written
/// twice (T-SQL and QuestDB SQL) and kept bit-identical; evaluating once in C# keeps the backends equal by
/// construction. Memory stays O(buckets): the rows are streamed, never materialised.
/// </remarks>
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
/// <returns>The SQL batch.</returns>
public static string TimeWeighted(string sample)
{
const string cols = "SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode";
const string usable = " AND StatusCode >= 0 AND ValueKind <> 0";
var key = $"FROM {sample} WHERE TagId = @id AND ";
return
$"SELECT TOP 1 {cols} {key}SourceTimeUtc < @start{usable} ORDER BY SourceTimeUtc DESC; " +
$"SELECT TOP 1 {cols} {key}SourceTimeUtc < @start ORDER BY SourceTimeUtc DESC; " +
$"SELECT TOP (@maxRows) {cols} {key}SourceTimeUtc >= @start AND SourceTimeUtc < @end ORDER BY SourceTimeUtc ASC; " +
$"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end ORDER BY SourceTimeUtc ASC; " +
$"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end{usable} ORDER BY SourceTimeUtc ASC;";
}
/// <summary>The <c>s</c> CTE: the window's samples with bucket index and the usable/good flags.</summary>
private static string Classified(string sample) =>
"WITH s AS (SELECT (DATEDIFF_BIG(microsecond, @start, SourceTimeUtc) * 10 " +
@@ -59,6 +59,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
private readonly TimeProvider _time;
private readonly HistorianHealthCounters _health = new();
private readonly int _maxTimeWeightedRawRows;
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
/// <param name="conn">Connection helper.</param>
@@ -66,10 +67,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
/// <param name="catalog">Tag catalog (shared; reads use only its read-only lookup).</param>
/// <param name="log">Optional logger.</param>
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (&lt;= 0 = unbounded).</param>
internal SqlServerHistorianDataSource(
SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap, SqlServerTagCatalog catalog, ILogger? log = null,
TimeProvider? timeProvider = null)
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
{
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
@@ -298,10 +301,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
/// <para>
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
/// <see cref="double"/> computed over <c>NumValue</c> (Boolean = 0/1). These are the simple per-sample
/// definitions, not Part 13's interval-boundary (interpolated bound / time-weighted) semantics.
/// definitions. The Part 13 time-weighted family (TimeAverage, TimeAverage2, Total, Total2, Interpolative, #671) is
/// not pushed down: the window's rows plus four bounding rows are streamed into the shared
/// <see cref="TimeWeightedAggregator"/>, which fills every interval (an empty one between bounds gets a value).
/// </para>
/// </remarks>
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the thirteen served.</exception>
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the seventeen served.</exception>
/// <exception cref="ArgumentOutOfRangeException">The window holds more than <see cref="MaxProcessedBuckets"/> intervals.</exception>
public async Task<HistoryReadResult> ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
@@ -316,6 +321,14 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
var now = _time.GetUtcNow().UtcDateTime;
var evaluated = new Dictionary<long, DataValueSnapshot>();
var tagId = plan.BucketCount == 0 ? null : await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
if (family == AggregateFamily.TimeWeighted)
{
var tw = new TimeWeightedAggregator(aggregate, plan);
if (tagId is { } twId) await StreamTimeWeightedAsync(twId, fullReference, plan, tw, cancellationToken).ConfigureAwait(false);
var twResult = tw.Finish(now);
RecordSuccess();
return new HistoryReadResult(twResult, ContinuationPoint: null);
}
if (tagId is { } id)
{
var sample = _conn.Q("Sample");
@@ -344,6 +357,11 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
RecordSuccess();
return new HistoryReadResult(result, ContinuationPoint: null);
}
catch (HistoryReadLimitExceededException)
{
// A client-request limit (the raw-row cap), not a backend fault: not counted as a failure.
throw;
}
catch (Exception ex)
{
RecordFailure(ex);
@@ -351,6 +369,34 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
}
}
/// <summary>
/// Streams the time-weighted batch (<see cref="SqlAggregateQueries.TimeWeighted"/>) into <paramref name="tw"/>: the
/// two before-bounds, the window, the two after-bounds — ascending, with duplicates dropped by the aggregator.
/// </summary>
private async Task StreamTimeWeightedAsync(int id, string tag, BucketPlan plan, TimeWeightedAggregator tw, CancellationToken ct)
{
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
await using var cmd = _conn.CreateCommand(conn, SqlAggregateQueries.TimeWeighted(_conn.Q("Sample")));
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
SqlHistorianConnection.AddDateTime2(cmd, "@start", DateTime.SpecifyKind(plan.StartUtc, DateTimeKind.Utc));
SqlHistorianConnection.AddDateTime2(cmd, "@end", DateTime.SpecifyKind(plan.EndUtc, DateTimeKind.Utc));
cmd.Parameters.Add(new SqlParameter("@maxRows", SqlDbType.Int)
{ Value = TimeWeightedAggregator.WindowFetchLimit(_maxTimeWeightedRawRows) });
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
var resultSet = 0;
do
{
long windowRows = 0;
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
// Result set 2 is the window (TOP (@maxRows) = cap + 1): one row past the cap fails the node.
if (resultSet == 2) TimeWeightedAggregator.CheckWindowRows(++windowRows, _maxTimeWeightedRawRows, tag);
tw.Add(ReadStored(reader, 0));
}
resultSet++;
} while (await reader.NextResultAsync(ct).ConfigureAwait(false));
}
private static BucketRow ReadBucket(SqlDataReader reader, AggregateFamily family)
{
static double? D(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
@@ -1,3 +1,4 @@
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
using System.Collections.Frozen;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
@@ -29,11 +30,16 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
/// <summary>Creates the provider over the <c>ServerHistorian:SqlServer</c> options (validated elsewhere).</summary>
/// <param name="options">The <c>ServerHistorian:SqlServer</c> options.</param>
public SqlServerHistorianProvider(SqlServerHistorianOptions options)
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (&lt;= 0 = unbounded).</param>
public SqlServerHistorianProvider(SqlServerHistorianOptions options,
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
}
private readonly int _maxTimeWeightedRawRows;
/// <inheritdoc />
public string Name => "SqlServer";
@@ -63,7 +69,7 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
{
var s = Get(sp);
return new SqlServerHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>());
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
}
/// <inheritdoc />
@@ -41,8 +41,8 @@ public static class HistorianBackendRegistry
new Dictionary<string, Func<ServerHistorianOptions, IConfiguration, IHistorianBackendProvider>>(StringComparer.OrdinalIgnoreCase)
{
[ServerHistorianOptions.GatewayBackendName] = (o, _) => new GatewayHistorianProvider(o),
[SqlServerBackendName] = (_, c) => new SqlServerHistorianProvider(BindSqlServer(c)),
[QuestDbBackendName] = (_, c) => new QuestDbHistorianProvider(BindQuestDb(c)),
[SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows),
[QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows),
};
/// <summary>The registered backend names, for validation messages.</summary>
@@ -5878,6 +5878,14 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
errors[handle.Index] = StatusCodes.BadAggregateNotSupported;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadAggregateNotSupported };
}
catch (HistoryReadLimitExceededException ex)
{
// The read would exceed a backend size limit (#671: the time-weighted raw-row cap) — refuse THIS node
// explicitly instead of a generic fault or a deadline-elapsed BadTimeout.
_logger.LogWarning("OtOpcUaNodeManager: HistoryRead limit exceeded for node {NodeId}: {Message}", handle.NodeId, ex.Message);
errors[handle.Index] = StatusCodes.BadTooManyOperations;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations };
}
catch (Exception ex)
{
// One node's backend failure (throw / timeout) must not throw out of the batch — surface a
@@ -6174,6 +6182,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
if (aggregateNodeId == ObjectIds.AggregateFunction_AnnotationCount) return HistoryAggregateType.AnnotationCount;
if (aggregateNodeId == ObjectIds.AggregateFunction_StandardDeviationPopulation) return HistoryAggregateType.StandardDeviationPopulation;
if (aggregateNodeId == ObjectIds.AggregateFunction_VariancePopulation) return HistoryAggregateType.VariancePopulation;
if (aggregateNodeId == ObjectIds.AggregateFunction_TimeAverage2) return HistoryAggregateType.TimeAverage2;
if (aggregateNodeId == ObjectIds.AggregateFunction_Total2) return HistoryAggregateType.Total2;
return null;
}
@@ -112,6 +112,16 @@ public sealed class ServerHistorianOptions
/// </summary>
public TimeSpan HistoryReadDeadline { get; init; } = TimeSpan.FromSeconds(60);
/// <summary>
/// <c>SqlServer</c> / <c>QuestDb</c> backends only (#671): the most raw in-window samples one Part 13
/// time-weighted read (TimeAverage, TimeAverage2, Total, Total2, Interpolative) may stream into the server-side
/// evaluator — that family is not pushed down to SQL. A read past the cap stops at cap + 1 rows and fails THAT
/// node with <c>BadTooManyOperations</c> instead of holding a pooled connection until
/// <see cref="HistoryReadDeadline"/>. Non-positive = unbounded (a <see cref="Validate"/> warning). Default
/// 1 000 000 (≈ 11.5 days of a 1 Hz tag).
/// </summary>
public int MaxTimeWeightedRawRows { get; init; } = 1_000_000;
/// <summary>Returns operator-facing misconfiguration warnings for an <c>Enabled</c> historian
/// (empty when disabled or correctly configured). Pure — the registration logs each entry.
/// <para>
@@ -154,6 +164,8 @@ public sealed class ServerHistorianOptions
warnings.Add($"ServerHistorian:HistoryReadBatchParallelism is {HistoryReadBatchParallelism} — must be > 0; HistoryRead falls back to serving each node sequentially.");
if (MaxConcurrentHistoryReadBatches <= 0)
warnings.Add($"ServerHistorian:MaxConcurrentHistoryReadBatches is {MaxConcurrentHistoryReadBatches} — must be > 0; the process-wide HistoryRead batch limiter is disabled.");
if (MaxTimeWeightedRawRows <= 0)
warnings.Add($"ServerHistorian:MaxTimeWeightedRawRows is {MaxTimeWeightedRawRows} — must be > 0; time-weighted aggregate reads on the SqlServer/QuestDb backends stream an unbounded number of raw samples.");
if (HistoryReadDeadline <= TimeSpan.Zero)
warnings.Add($"ServerHistorian:HistoryReadDeadline is {HistoryReadDeadline} — must be > 0; HistoryRead requests run unbounded (no per-request deadline).");
return warnings;