fix(alarms): never drop a comment-trail entry on pair divergence (E5 review r1)
I-1: a node whose in-memory trail is one short of the stored trail (it missed a command the peer applied, whose entry then replicated in) silently dropped its own next entry — in failover, the survivor's newest ack — because the positional "past MAX(seq)" rule treated it as already stored. SaveAsync now reconciles the last 16 in-memory entries against the stored rows from that position on, by content (Kind/User/Text, timestamps within a 5-minute skew window; matched rows are consumed so repeats count). The peer's copy of the same command is recognised and not re-appended; an unmatched entry is inserted at its own ordinal when free, otherwise appended at MAX(seq)+1 with a Warning — kept, never silent. Bounded read, constant write. m1: MigrateInlineComments copies only — it no longer UPDATEs the condition row (a fresh-HLC re-stamp could beat a newer peer ack under LWW); the next save clears comments_json. m2: it returns unparseable alarm ids and OnReady logs a Warning per id (+ an Information tally). m3/m4: documented in the store remarks (Remove vs lagging peer; in-memory index == durable ordinal). Tests: store-level failover sequence (missed command -> survivor's ack and next entry persisted once, re-save writes 0 rows, one Warning) with the dual-write positive control (peer copy present -> 0 rows, no Warning) and a repeated-identical-comment case; a loopback pair failover test (link down, survivor acks, rejoin -> both entries on both peers, held); migration no-row-rewrite, partial fill and unparseable reporting; OnReady Warning. Both failover tests fail against the round-0 store. Refs #748 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
@@ -46,7 +46,8 @@ namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
||||
/// O(1) however long the history is. <c>comments_json</c> is kept, always written as
|
||||
/// <c>"[]"</c>, so the condition table's shape — and with it the pair's replication digest
|
||||
/// for that table — does not change and an old-shape row stays readable;
|
||||
/// <see cref="MigrateInlineComments"/> moves any inline trail across.
|
||||
/// <see cref="MigrateInlineComments"/> copies any inline trail across (the alarm's next
|
||||
/// save clears the column).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why the ordinal, and not a content hash, is the key.</b> Both halves of a redundant pair
|
||||
@@ -55,7 +56,9 @@ namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
||||
/// converged. Keyed by ordinal, the two nodes' copies of entry <i>n</i> share one primary key
|
||||
/// and LWW collapses them to one row — the convergence the whole-row LWW gave before, now per
|
||||
/// entry. Entries go in with <c>INSERT OR IGNORE</c> (<see cref="InsertCommentSql"/>), so an
|
||||
/// entry already held — written locally, or arrived from the peer — is never rewritten.
|
||||
/// entry already held — written locally, or arrived from the peer — is never rewritten. A
|
||||
/// node that MISSED a peer command keeps its own colliding entry by appending it after the
|
||||
/// stored trail, with a Warning (see the store's divergence handling).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class AlarmConditionStateSchema
|
||||
@@ -100,8 +103,8 @@ public static class AlarmConditionStateSchema
|
||||
// Acknowledged/Unacknowledged, Confirmed/Unconfirmed, Unshelved/OneShotShelved/TimedShelved),
|
||||
// so they are NOT NULL. comments_json is never null either — it is the literal "[]" since the
|
||||
// trail moved to alarm_condition_comment (an old-shape row may still carry an inline trail
|
||||
// until MigrateInlineComments or the next save moves it). The audit user/comment/utc columns
|
||||
// and shelving_expires_utc are NULL when the alarm has no such history yet. updated_at_utc
|
||||
// until the alarm's next save; MigrateInlineComments copies it without touching this row).
|
||||
// The audit user/comment/utc columns and shelving_expires_utc are NULL when the alarm has no such history yet. updated_at_utc
|
||||
// carries LastTransitionUtc (there is no dedicated transition column).
|
||||
cmd.CommandText = """
|
||||
CREATE TABLE IF NOT EXISTS alarm_condition_state (
|
||||
@@ -134,9 +137,17 @@ public static class AlarmConditionStateSchema
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
/// <summary>What one <see cref="MigrateInlineComments"/> run did.</summary>
|
||||
/// <param name="RowsMoved">Condition rows for which at least one inline entry was copied across.</param>
|
||||
/// <param name="UnparseableAlarmIds">
|
||||
/// Alarms whose inline trail is not valid JSON and was left where it is — the caller logs these,
|
||||
/// so an unmigratable history is visible rather than skipped silently on every boot.
|
||||
/// </param>
|
||||
public sealed record InlineCommentMigrationResult(int RowsMoved, IReadOnlyList<string> UnparseableAlarmIds);
|
||||
|
||||
/// <summary>
|
||||
/// Moves every condition row's inline <c>comments_json</c> trail into
|
||||
/// <see cref="CommentsTable"/> and resets the column to <c>"[]"</c>. Idempotent.
|
||||
/// Copies every condition row's inline <c>comments_json</c> trail into
|
||||
/// <see cref="CommentsTable"/>. Idempotent; it never writes the condition row itself.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
@@ -145,18 +156,26 @@ public static class AlarmConditionStateSchema
|
||||
/// DDL → register → writes rule <c>AlarmSfLegacyMigrator</c> follows).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Copy, don't clear.</b> Resetting <c>comments_json</c> here would be a full-row UPDATE
|
||||
/// that re-stamps the condition row with a fresh HLC — during a pair's rolling upgrade
|
||||
/// that stale-but-newly-stamped copy would beat a newer peer state (an ack made on the
|
||||
/// already-upgraded node) under LWW (review m1). The column is left as it is: the store's
|
||||
/// load merges an inline trail with the table by ordinal (the table wins), and the
|
||||
/// engine's next save of that alarm writes <c>"[]"</c> with the alarm's real state.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Each entry lands at its ordinal in the inline array via <see cref="InsertCommentSql"/>,
|
||||
/// so a re-run — or a row whose trail was already (partly) moved — neither duplicates nor
|
||||
/// reorders anything, and a second run finds no non-empty inline column and writes
|
||||
/// nothing. Each row moves in its own transaction together with its column reset, so a
|
||||
/// crash leaves every row either fully moved or untouched. A row whose inline JSON does
|
||||
/// not parse is left exactly as it is (the pre-split store could not load it either)
|
||||
/// rather than failing host start.
|
||||
/// so a re-run — or a row whose trail was already (partly) copied — neither duplicates nor
|
||||
/// reorders anything and writes nothing for an ordinal already held. Each row copies in its
|
||||
/// own transaction, so a crash leaves it fully copied or untouched. A row whose inline JSON
|
||||
/// does not parse is left exactly as it is (the pre-split store could not load it either)
|
||||
/// and reported in <see cref="InlineCommentMigrationResult.UnparseableAlarmIds"/> rather
|
||||
/// than failing host start.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="connection">An already-open connection from <c>ILocalDb.CreateConnection()</c>.</param>
|
||||
/// <returns>The number of condition rows whose inline trail was moved.</returns>
|
||||
public static int MigrateInlineComments(SqliteConnection connection)
|
||||
/// <returns>What was copied, and which rows could not be read.</returns>
|
||||
public static InlineCommentMigrationResult MigrateInlineComments(SqliteConnection connection)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connection);
|
||||
|
||||
@@ -172,6 +191,7 @@ public static class AlarmConditionStateSchema
|
||||
}
|
||||
|
||||
var migrated = 0;
|
||||
var unparseable = new List<string>();
|
||||
foreach (var (alarmId, json) in pending)
|
||||
{
|
||||
IReadOnlyList<InlineComment> entries;
|
||||
@@ -181,9 +201,11 @@ public static class AlarmConditionStateSchema
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
unparseable.Add(alarmId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var inserted = 0;
|
||||
using var tx = connection.BeginTransaction();
|
||||
for (var seq = 0; seq < entries.Count; seq++)
|
||||
{
|
||||
@@ -196,24 +218,15 @@ public static class AlarmConditionStateSchema
|
||||
insert.Parameters.AddWithValue("@User", entries[seq].User);
|
||||
insert.Parameters.AddWithValue("@Kind", entries[seq].Kind);
|
||||
insert.Parameters.AddWithValue("@Text", entries[seq].Text);
|
||||
insert.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
using (var clear = connection.CreateCommand())
|
||||
{
|
||||
clear.Transaction = tx;
|
||||
clear.CommandText =
|
||||
"UPDATE alarm_condition_state SET comments_json = @Empty WHERE scripted_alarm_id = @Id";
|
||||
clear.Parameters.AddWithValue("@Empty", EmptyInlineComments);
|
||||
clear.Parameters.AddWithValue("@Id", alarmId);
|
||||
clear.ExecuteNonQuery();
|
||||
inserted += insert.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
tx.Commit();
|
||||
migrated++;
|
||||
if (inserted > 0)
|
||||
migrated++;
|
||||
}
|
||||
|
||||
return migrated;
|
||||
return new InlineCommentMigrationResult(migrated, unparseable);
|
||||
}
|
||||
|
||||
/// <summary>Formats a comment timestamp for <see cref="CommentsTable"/> (round-trip "O" text).</summary>
|
||||
|
||||
Reference in New Issue
Block a user