Joseph Doherty
cfd403710e
Every HistoryUpdate a SQL backend applies now leaves a modification trail
(prior value, principal, Part 11 update type, modification time), and
HistoryReadRaw with IsReadModified=true serves it as HistoryModifiedData.
- Core: HistoryUpdateContext threaded into all three IHistorianDataUpdater
methods; IHistorianDataSource.ReadModifiedAsync (default NotSupported =>
BadHistoryOperationUnsupported on Gateway/Null); HistoryModified* contracts.
- SqlServer: schema v2 (002_sample_modified.sql, RequiredVersion=2, docs copy);
trail written by OUTPUT ... INTO on the same MERGE/DELETE statement, so a
failed update leaves no audit row; retention purges SampleModified per tag on
the source-time cutoff; ReadModifiedAsync keyed on (SourceTimeUtc, ModifiedId).
- QuestDB: append-only {prefix}sample_modified (TTL like samples), trail row
written BEFORE the change (orphan-on-crash, never an untracked change);
existence reads now carry the prior value; ReadModifiedAsync.
- Node manager: modified reads dispatch + page via the #669 continuation store
(HistoryContinuationState gains HistoryCursorKind + LastModifiedId), gated by
the existing HistoryRead decision; session principal / key id passed to the
updater. IsDeleteModified stays unsupported.
- Client.CLI: historyread --modified.
- Tests: node-manager dispatch/paging/cursor binding/principal threading,
over-the-wire HistoryModifiedData, shared ModifiedValueConformance on both
backends, SqlServer atomicity (forced trail failure => no change) + v1->v2
upgrade + retention trail purge.
- Docs: Historian.md (Modified-value history section, schema v2 upgrade note),
security.md, Client.CLI.md, CLAUDE.md.
432 lines
23 KiB
C#
432 lines
23 KiB
C#
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Npgsql;
|
|
using NpgsqlTypes;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
|
|
|
/// <summary>
|
|
/// OPC UA HistoryUpdate against the QuestDB sample table (spec §2.3). Every operation is an <c>INSERT</c>: the
|
|
/// table's <c>DEDUP UPSERT KEYS(ts, tag)</c> performs the replace, and a delete writes a tombstone
|
|
/// (<c>deleted = true</c>, NULL values, <c>BadNoData</c>). QuestDB <c>UPDATE</c>/<c>DELETE</c> are never issued.
|
|
/// <para>Every status comes from <see cref="HistoryUpdatePlanner"/>, so codes equal the SQL Server backend's.
|
|
/// "Exists" means present and not deleted, so Insert over a tombstone succeeds (<c>GoodEntryInserted</c>) while
|
|
/// Replace and DeleteAtTime treat a tombstone as no entry. Existence is read with one range query and filtered
|
|
/// client-side (never an IN list).</para>
|
|
/// <para><b>Concurrency:</b> calls are serialized per tag inside this instance (64 striped async locks held from
|
|
/// the existence read through the visibility wait), so concurrent Insert/Replace/Delete on one tag cannot
|
|
/// both act on a stale read. This does NOT cover the live writer racing on the same key, nor a second node;
|
|
/// that residual is accepted.</para>
|
|
/// <para><b>Pre-1970 (RC1):</b> QuestDB cannot store such a designated timestamp. Insert/Replace/Update of such an
|
|
/// item reports <c>BadOutOfRange</c> and sends nothing (a null/<c>MinValue</c> timestamp is still
|
|
/// <c>BadInvalidTimestamp</c>); DeleteAtTime reports <c>BadNoEntryExists</c>; a DeleteRaw window may start before
|
|
/// 1970 (read bounds are not clamped).</para>
|
|
/// <para><b>Read-your-writes:</b> after writing, <see cref="QuestDbWalVisibility"/> waits for the rows to become
|
|
/// visible, bounded by <c>VisibilityTimeout</c>; a timeout only logs a Warning, never changes a code.</para>
|
|
/// <para><b>Modified-value history (Part 11, #668).</b> Every sample this updater inserts, replaces or tombstones first
|
|
/// gets a row in <c>{prefix}sample_modified</c> carrying its PRIOR live value (the existence read already fetches it;
|
|
/// none for an insert — a tombstone is "absent", so an insert over one records no prior value), the caller's
|
|
/// <see cref="HistoryUpdateContext"/>, the <see cref="HistoryModificationType"/> and the modification time. QuestDB has
|
|
/// no multi-table transaction, so the trail row is written <b>BEFORE</b> the change: a crash (or a failed sample INSERT)
|
|
/// between the two leaves an orphan trail row describing a change that never landed — never a change without its
|
|
/// trail row. This is the documented deviation from SQL Server, whose trail commits atomically with the change.</para>
|
|
/// </summary>
|
|
internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
|
{
|
|
/// <summary>Rows per INSERT and per DeleteRaw page (T0 Q2: 9,000 parameters, far under the 32,767 limit).</summary>
|
|
internal const int BatchSize = 1000;
|
|
|
|
private const long ResolutionTicks = 10;
|
|
private const int LockStripes = 64;
|
|
private readonly SemaphoreSlim[] _locks = Enumerable.Range(0, LockStripes).Select(_ => new SemaphoreSlim(1, 1)).ToArray();
|
|
private long _liveRowsRead;
|
|
private static long _lastModifiedId;
|
|
private readonly QuestDbConnection _conn;
|
|
private readonly QuestDbTagCatalog _tags;
|
|
private readonly QuestDbWalVisibility _wal;
|
|
private readonly ILogger _log;
|
|
|
|
/// <summary>Creates the updater.</summary>
|
|
/// <param name="conn">Connection helper.</param>
|
|
/// <param name="tags">Tag catalog (R7: the only known-tag authority).</param>
|
|
/// <param name="wal">WAL visibility wait (R5).</param>
|
|
/// <param name="log">Optional logger.</param>
|
|
public QuestDbHistorianDataUpdater(QuestDbConnection conn, QuestDbTagCatalog tags, QuestDbWalVisibility wal,
|
|
ILogger? log = null)
|
|
{
|
|
_conn = conn;
|
|
_tags = tags;
|
|
_wal = wal;
|
|
_log = log ?? NullLogger.Instance;
|
|
}
|
|
|
|
/// <summary>Rows returned by existence reads so far (test/diagnostic counter).</summary>
|
|
internal long LiveRowsRead => Interlocked.Read(ref _liveRowsRead);
|
|
|
|
private async Task<IDisposable> LockAsync(string tag, CancellationToken ct)
|
|
{
|
|
var sem = _locks[(int)((uint)StringComparer.Ordinal.GetHashCode(tag) % LockStripes)];
|
|
await sem.WaitAsync(ct).ConfigureAwait(false);
|
|
return new Releaser(sem);
|
|
}
|
|
|
|
private sealed class Releaser(SemaphoreSlim sem) : IDisposable
|
|
{
|
|
public void Dispose() => sem.Release();
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public async Task<IReadOnlyList<uint>> UpdateDataAsync(string tag, HistoryUpdateMode mode,
|
|
IReadOnlyList<HistorizationValue> values, HistoryUpdateContext context, CancellationToken ct)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(tag);
|
|
ArgumentNullException.ThrowIfNull(values);
|
|
ArgumentNullException.ThrowIfNull(context);
|
|
|
|
// RC1: a valid-timestamp item before 1970 can never be stored; refuse it and plan the rest.
|
|
var final = new uint[values.Count];
|
|
var kept = new List<int>(values.Count);
|
|
for (var i = 0; i < values.Count; i++)
|
|
{
|
|
if (values[i].TimestampUtc is { } ts && ts != DateTime.MinValue && !QuestDbConnection.IsStorable(ts))
|
|
final[i] = HistorianStatusCodes.BadOutOfRange;
|
|
else
|
|
kept.Add(i);
|
|
}
|
|
|
|
var plan = HistoryUpdatePlanner.Plan(tag, mode, kept.Select(i => values[i]).ToList(), ResolutionTicks);
|
|
IReadOnlyList<uint> planned;
|
|
if (plan.Winners.Count == 0)
|
|
{
|
|
planned = HistoryUpdatePlanner.Complete(plan, mode, new Dictionary<long, UpsertOutcome>());
|
|
}
|
|
else
|
|
{
|
|
using var _ = await LockAsync(tag, ct).ConfigureAwait(false);
|
|
var known = mode == HistoryUpdateMode.Replace
|
|
? await _tags.ExistsAsync(tag, ct).ConfigureAwait(false)
|
|
: await EnsureAsync(tag, plan.Winners[0].Row.Value.Kind, ct).ConfigureAwait(false);
|
|
var outcome = known
|
|
? await ApplyAsync(tag, mode, plan.Winners.Select(w => w.Row).ToList(), context, ct).ConfigureAwait(false)
|
|
: new Dictionary<long, UpsertOutcome>();
|
|
planned = HistoryUpdatePlanner.Complete(plan, mode, outcome);
|
|
}
|
|
for (var k = 0; k < kept.Count; k++) final[kept[k]] = planned[k];
|
|
return final;
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public async Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, HistoryUpdateContext context,
|
|
CancellationToken ct)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(tag);
|
|
ArgumentNullException.ThrowIfNull(context);
|
|
if (!HistorianTagNames.IsValid(tag)) return HistorianStatusCodes.BadInvalidArgument;
|
|
if (!await _tags.ExistsAsync(tag, ct).ConfigureAwait(false))
|
|
return HistoryUpdatePlanner.DeleteRawResult(tagKnown: false, 0);
|
|
|
|
using var _ = await LockAsync(tag, ct).ConfigureAwait(false);
|
|
var cursor = QuestDbConnection.Round(startUtc);
|
|
var end = QuestDbConnection.Round(endUtc);
|
|
long total = 0;
|
|
while (cursor < end)
|
|
{
|
|
var page = new List<DateTime>(BatchSize);
|
|
var priors = new List<LiveRow>(BatchSize);
|
|
await using (var conn = await _conn.OpenAsync(ct).ConfigureAwait(false))
|
|
await using (var cmd = _conn.CreateCommand(conn,
|
|
$"SELECT {LiveColumns} FROM {_conn.SampleTable} WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts LIMIT {BatchSize}"))
|
|
{
|
|
QuestDbConnection.BindText(cmd, tag);
|
|
QuestDbConnection.BindTimestamp(cmd, cursor);
|
|
QuestDbConnection.BindTimestamp(cmd, end);
|
|
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
|
while (await rd.ReadAsync(ct).ConfigureAwait(false))
|
|
{
|
|
var row = ReadLiveRow(rd);
|
|
page.Add(row.TimeUtc);
|
|
priors.Add(row);
|
|
}
|
|
}
|
|
if (page.Count == 0) break;
|
|
// Trail BEFORE the change (no multi-table transaction): a crash here leaves an orphan trail row, never a
|
|
// tombstone without its trail row.
|
|
await WriteTrailAsync(tag, HistoryModificationType.Delete,
|
|
priors.Select(r => (r.TimeUtc, (LiveRow?)r)).ToList(), context, ct).ConfigureAwait(false);
|
|
await WriteTombstonesAsync(tag, page, ct).ConfigureAwait(false);
|
|
// The wait between pages is required: without it the next page could re-read rows just tombstoned.
|
|
await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false);
|
|
total += page.Count;
|
|
if (page.Count < BatchSize) break;
|
|
cursor = page[^1].AddTicks(ResolutionTicks);
|
|
}
|
|
return HistoryUpdatePlanner.DeleteRawResult(tagKnown: true, total);
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public async Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc,
|
|
HistoryUpdateContext context, CancellationToken ct)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(tag);
|
|
ArgumentNullException.ThrowIfNull(timesUtc);
|
|
ArgumentNullException.ThrowIfNull(context);
|
|
if (!HistorianTagNames.IsValid(tag))
|
|
{
|
|
var refused = new uint[timesUtc.Count];
|
|
Array.Fill(refused, HistorianStatusCodes.BadInvalidArgument);
|
|
return refused;
|
|
}
|
|
|
|
// M-2: round BEFORE both the delete and the comparison, so a sub-µs request time matches the row it deleted.
|
|
// MinValue stays as is: the planner turns it into BadInvalidTimestamp.
|
|
var rounded = timesUtc.Select(t => t == DateTime.MinValue ? t : QuestDbConnection.Round(t)).ToList();
|
|
var candidates = new HashSet<long>();
|
|
foreach (var t in rounded)
|
|
if (t != DateTime.MinValue && t >= DateTime.UnixEpoch) candidates.Add(t.Ticks); // pre-1970 can hold no row
|
|
var deleted = new HashSet<long>();
|
|
if (candidates.Count == 0 || !await _tags.ExistsAsync(tag, ct).ConfigureAwait(false))
|
|
return HistoryUpdatePlanner.DeleteAtTimeResults(rounded, deleted, tagKnown: false);
|
|
|
|
using var _ = await LockAsync(tag, ct).ConfigureAwait(false);
|
|
var live = await ReadLiveAsync(tag, candidates.Select(t => new DateTime(t, DateTimeKind.Utc)).ToList(), ct).ConfigureAwait(false);
|
|
var toDelete = candidates.Where(live.ContainsKey).OrderBy(x => x).ToList();
|
|
foreach (var chunk in toDelete.Chunk(BatchSize))
|
|
{
|
|
// Trail BEFORE the change (see the class remarks).
|
|
await WriteTrailAsync(tag, HistoryModificationType.Delete,
|
|
chunk.Select(t => (live[t].TimeUtc, (LiveRow?)live[t])).ToList(), context, ct).ConfigureAwait(false);
|
|
await WriteTombstonesAsync(tag, chunk.Select(t => new DateTime(t, DateTimeKind.Utc)).ToList(), ct).ConfigureAwait(false);
|
|
}
|
|
if (toDelete.Count > 0) await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false);
|
|
deleted.UnionWith(toDelete);
|
|
return HistoryUpdatePlanner.DeleteAtTimeResults(rounded, deleted, tagKnown: true);
|
|
}
|
|
|
|
private async Task<bool> EnsureAsync(string tag, HistorianValueKind kind, CancellationToken ct)
|
|
{
|
|
await _tags.EnsureAsync(tag, HistorianDataTypes.FromKind(kind), ct).ConfigureAwait(false);
|
|
return true;
|
|
}
|
|
|
|
private async Task<Dictionary<long, UpsertOutcome>> ApplyAsync(string tag, HistoryUpdateMode mode,
|
|
List<HistorianSampleRow> rows, HistoryUpdateContext context, CancellationToken ct)
|
|
{
|
|
// Key = the planner's tick (what Complete looks up); the stored time is Round(of the same row).
|
|
var live = await ReadLiveAsync(tag, rows.Select(r => QuestDbConnection.Round(r.TimeUtc)).ToList(), ct).ConfigureAwait(false);
|
|
var outcome = new Dictionary<long, UpsertOutcome>();
|
|
var toWrite = new List<HistorianSampleRow>();
|
|
var trail = new List<(DateTime, LiveRow?)>();
|
|
foreach (var r in rows)
|
|
{
|
|
var stored = QuestDbConnection.Round(r.TimeUtc);
|
|
var present = live.TryGetValue(stored.Ticks, out var prior);
|
|
if (mode == HistoryUpdateMode.Insert && present) continue;
|
|
if (mode == HistoryUpdateMode.Replace && !present) continue;
|
|
outcome[r.TimeUtc.Ticks] = present ? UpsertOutcome.Replaced : UpsertOutcome.Inserted;
|
|
toWrite.Add(r);
|
|
trail.Add((stored, present ? prior : null));
|
|
}
|
|
// Trail BEFORE the change (see the class remarks).
|
|
await WriteTrailAsync(tag, ToModificationType(mode), trail, context, ct).ConfigureAwait(false);
|
|
foreach (var chunk in toWrite.Chunk(BatchSize))
|
|
await WriteRowsAsync(tag, chunk, ct).ConfigureAwait(false);
|
|
if (toWrite.Count > 0) await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false);
|
|
return outcome;
|
|
}
|
|
|
|
/// <summary>The live (present, not deleted) rows near the given times, keyed by stored ticks: one range query per cluster (I-2).</summary>
|
|
private async Task<Dictionary<long, LiveRow>> ReadLiveAsync(string tag, IReadOnlyList<DateTime> times, CancellationToken ct)
|
|
{
|
|
var live = new Dictionary<long, LiveRow>();
|
|
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
|
foreach (var (lo, hi) in QuestDbTimeClusters.Cluster(times, QuestDbTimeClusters.ExistenceGap))
|
|
{
|
|
await using var cmd = _conn.CreateCommand(conn,
|
|
$"SELECT {LiveColumns} FROM {_conn.SampleTable} WHERE tag = $1 AND ts >= $2 AND ts <= $3 AND NOT deleted");
|
|
QuestDbConnection.BindText(cmd, tag);
|
|
QuestDbConnection.BindTimestamp(cmd, lo);
|
|
QuestDbConnection.BindTimestamp(cmd, hi);
|
|
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
|
while (await rd.ReadAsync(ct).ConfigureAwait(false))
|
|
{
|
|
var row = ReadLiveRow(rd);
|
|
live[row.TimeUtc.Ticks] = row;
|
|
Interlocked.Increment(ref _liveRowsRead);
|
|
}
|
|
}
|
|
return live;
|
|
}
|
|
|
|
/// <summary>The columns of a live sample the trail needs (prior value + status + server time).</summary>
|
|
private const string LiveColumns = "ts, kind, num_value, int_value, str_value, status, server_ts";
|
|
|
|
/// <summary>A live sample as it was before a modification.</summary>
|
|
private readonly record struct LiveRow(DateTime TimeUtc, HistorianValue Value, uint Status, DateTime? ServerTimeUtc);
|
|
|
|
private static LiveRow ReadLiveRow(NpgsqlDataReader rd)
|
|
{
|
|
var value = QuestDbHistorianDataSource.ToValue(
|
|
QuestDbConnection.ReadKind(rd, 1),
|
|
rd.IsDBNull(2) ? null : rd.GetDouble(2),
|
|
rd.IsDBNull(3) ? null : rd.GetInt64(3),
|
|
rd.IsDBNull(4) ? null : rd.GetString(4));
|
|
var status = rd.IsDBNull(5) ? HistorianStatusCodes.BadNoData : SampleColumnMapper.FromLongStatus(rd.GetInt64(5));
|
|
return new LiveRow(QuestDbConnection.ReadUtc(rd, 0), value, status,
|
|
rd.IsDBNull(6) ? null : QuestDbConnection.ReadUtc(rd, 6));
|
|
}
|
|
|
|
/// <summary>The Part 11 modification type recorded for an update mode.</summary>
|
|
private static HistoryModificationType ToModificationType(HistoryUpdateMode mode) => mode switch
|
|
{
|
|
HistoryUpdateMode.Insert => HistoryModificationType.Insert,
|
|
HistoryUpdateMode.Replace => HistoryModificationType.Replace,
|
|
_ => HistoryModificationType.Update,
|
|
};
|
|
|
|
/// <summary>
|
|
/// A process-wide, strictly increasing modification id (≈ UTC ticks, bumped past the last issued value). It orders a
|
|
/// tag's trail within one source time; ids from two nodes interleave by wall clock, which is the modification order.
|
|
/// </summary>
|
|
internal static long NextModifiedId()
|
|
{
|
|
while (true)
|
|
{
|
|
var last = Interlocked.Read(ref _lastModifiedId);
|
|
var next = Math.Max(DateTime.UtcNow.Ticks, last + 1);
|
|
if (Interlocked.CompareExchange(ref _lastModifiedId, next, last) == last) return next;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Appends one modification-trail row per entry (<c>prior = null</c> ⇒ an insert: kind Empty, NULL value and
|
|
/// status), then waits for the trail's WAL visibility so a modified read right after the update sees it.
|
|
/// </summary>
|
|
private async Task WriteTrailAsync(string tag, HistoryModificationType type, IReadOnlyList<(DateTime TimeUtc, LiveRow? Prior)> entries,
|
|
HistoryUpdateContext context, CancellationToken ct)
|
|
{
|
|
if (entries.Count == 0) return;
|
|
var now = DateTime.UtcNow;
|
|
foreach (var chunk in entries.Chunk(BatchSize))
|
|
{
|
|
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
|
await using var cmd = _conn.CreateCommand(conn, "");
|
|
var sb = new System.Text.StringBuilder(
|
|
$"INSERT INTO {_conn.SampleModifiedTable} (ts, tag, modified_id, modification_ts, update_type, principal, " +
|
|
"api_key_id, kind, num_value, int_value, str_value, status, server_ts) VALUES ");
|
|
for (var i = 0; i < chunk.Length; i++)
|
|
{
|
|
var (time, prior) = chunk[i];
|
|
QuestDbConnection.BindTimestamp(cmd, time, clampToEpoch: true);
|
|
QuestDbConnection.BindText(cmd, tag);
|
|
QuestDbConnection.BindLong(cmd, NextModifiedId());
|
|
QuestDbConnection.BindTimestamp(cmd, now);
|
|
QuestDbConnection.BindByte(cmd, (byte)type);
|
|
QuestDbConnection.BindText(cmd, context.StoredPrincipal);
|
|
QuestDbConnection.BindText(cmd, context.StoredApiKeyId);
|
|
if (prior is { } p)
|
|
{
|
|
var c = SampleColumnMapper.ToColumns(p.Value);
|
|
QuestDbConnection.BindByte(cmd, c.ValueKind);
|
|
Typed(cmd, NpgsqlDbType.Double, c.NumValue);
|
|
Typed(cmd, NpgsqlDbType.Bigint, c.IntValue);
|
|
Typed(cmd, NpgsqlDbType.Varchar, c.StrValue);
|
|
QuestDbConnection.BindLong(cmd, SampleColumnMapper.ToLongStatus(p.Status));
|
|
Typed(cmd, NpgsqlDbType.TimestampTz, p.ServerTimeUtc);
|
|
}
|
|
else
|
|
{
|
|
// No prior value: QuestDB BYTE has no NULL, so kind is Empty (0) and the NULL status marks "none".
|
|
QuestDbConnection.BindByte(cmd, (byte)HistorianValueKind.Empty);
|
|
Typed(cmd, NpgsqlDbType.Double, null);
|
|
Typed(cmd, NpgsqlDbType.Bigint, null);
|
|
Typed(cmd, NpgsqlDbType.Varchar, null);
|
|
Typed(cmd, NpgsqlDbType.Bigint, null);
|
|
Typed(cmd, NpgsqlDbType.TimestampTz, null);
|
|
}
|
|
|
|
var b = i * 13;
|
|
if (i > 0) sb.Append(',');
|
|
sb.Append('(');
|
|
for (var q = 1; q <= 13; q++) sb.Append(q > 1 ? "," : "").Append('$').Append(b + q);
|
|
sb.Append(')');
|
|
}
|
|
cmd.CommandText = sb.ToString();
|
|
try
|
|
{
|
|
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
_log.LogWarning(ex, "QuestDB modification-trail INSERT of {Count} rows into {Table} failed; the update was not applied.",
|
|
chunk.Length, _conn.SampleModifiedTable);
|
|
throw;
|
|
}
|
|
}
|
|
await _wal.WaitAsync(_conn.SampleModifiedTable, ct).ConfigureAwait(false);
|
|
}
|
|
|
|
private Task WriteRowsAsync(string tag, IReadOnlyList<HistorianSampleRow> rows, CancellationToken ct) =>
|
|
InsertAsync(rows.Count, ct, (cmd, i) =>
|
|
{
|
|
var row = rows[i];
|
|
var c = SampleColumnMapper.ToColumns(row.Value);
|
|
QuestDbConnection.BindTimestamp(cmd, row.TimeUtc, clampToEpoch: true);
|
|
QuestDbConnection.BindText(cmd, tag);
|
|
QuestDbConnection.BindByte(cmd, c.ValueKind);
|
|
Typed(cmd, NpgsqlDbType.Double, c.NumValue);
|
|
Typed(cmd, NpgsqlDbType.Bigint, c.IntValue);
|
|
Typed(cmd, NpgsqlDbType.Varchar, c.StrValue);
|
|
QuestDbConnection.BindLong(cmd, SampleColumnMapper.ToLongStatus(row.StatusCode));
|
|
Typed(cmd, NpgsqlDbType.Boolean, false);
|
|
});
|
|
|
|
private Task WriteTombstonesAsync(string tag, IReadOnlyList<DateTime> times, CancellationToken ct) =>
|
|
InsertAsync(times.Count, ct, (cmd, i) =>
|
|
{
|
|
QuestDbConnection.BindTimestamp(cmd, times[i], clampToEpoch: true);
|
|
QuestDbConnection.BindText(cmd, tag);
|
|
QuestDbConnection.BindByte(cmd, (byte)HistorianValueKind.Empty);
|
|
Typed(cmd, NpgsqlDbType.Double, null);
|
|
Typed(cmd, NpgsqlDbType.Bigint, null);
|
|
Typed(cmd, NpgsqlDbType.Varchar, null);
|
|
QuestDbConnection.BindLong(cmd, SampleColumnMapper.ToLongStatus(HistorianStatusCodes.BadNoData));
|
|
Typed(cmd, NpgsqlDbType.Boolean, true);
|
|
});
|
|
|
|
/// <summary>Multi-row INSERT; <paramref name="bindRow"/> adds the first eight parameters of a row, this adds server_ts.</summary>
|
|
private async Task InsertAsync(int count, CancellationToken ct, Action<NpgsqlCommand, int> bindRow)
|
|
{
|
|
if (count == 0) return;
|
|
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
|
await using var cmd = _conn.CreateCommand(conn, "");
|
|
var now = DateTime.UtcNow;
|
|
var sb = new System.Text.StringBuilder(
|
|
$"INSERT INTO {_conn.SampleTable} (ts, tag, kind, num_value, int_value, str_value, status, deleted, server_ts) VALUES ");
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
bindRow(cmd, i);
|
|
QuestDbConnection.BindTimestamp(cmd, now);
|
|
var b = i * 9;
|
|
if (i > 0) sb.Append(',');
|
|
sb.Append('(');
|
|
for (var p = 1; p <= 9; p++) sb.Append(p > 1 ? "," : "").Append('$').Append(b + p);
|
|
sb.Append(')');
|
|
}
|
|
cmd.CommandText = sb.ToString();
|
|
try
|
|
{
|
|
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
_log.LogWarning(ex, "QuestDB HistoryUpdate INSERT of {Count} rows into {Table} failed.", count, _conn.SampleTable);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private static void Typed(NpgsqlCommand cmd, NpgsqlDbType type, object? value) =>
|
|
cmd.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = type, Value = value ?? DBNull.Value });
|
|
}
|