Joseph Doherty
dad28d37a2
I-1: a node whose in-memory trail is one short of the stored trail (it missed a command the peer applied, whose entry then replicated in) silently dropped its own next entry — in failover, the survivor's newest ack — because the positional "past MAX(seq)" rule treated it as already stored. SaveAsync now reconciles the last 16 in-memory entries against the stored rows from that position on, by content (Kind/User/Text, timestamps within a 5-minute skew window; matched rows are consumed so repeats count). The peer's copy of the same command is recognised and not re-appended; an unmatched entry is inserted at its own ordinal when free, otherwise appended at MAX(seq)+1 with a Warning — kept, never silent. Bounded read, constant write. m1: MigrateInlineComments copies only — it no longer UPDATEs the condition row (a fresh-HLC re-stamp could beat a newer peer ack under LWW); the next save clears comments_json. m2: it returns unparseable alarm ids and OnReady logs a Warning per id (+ an Information tally). m3/m4: documented in the store remarks (Remove vs lagging peer; in-memory index == durable ordinal). Tests: store-level failover sequence (missed command -> survivor's ack and next entry persisted once, re-save writes 0 rows, one Warning) with the dual-write positive control (peer copy present -> 0 rows, no Warning) and a repeated-identical-comment case; a loopback pair failover test (link down, survivor acks, rejoin -> both entries on both peers, held); migration no-row-rewrite, partial fill and unparseable reporting; OnReady Warning. Both failover tests fail against the round-0 store. Refs #748 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
524 lines
25 KiB
C#
524 lines
25 KiB
C#
using System.Collections.Immutable;
|
|
using Microsoft.Data.Sqlite;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.LocalDb;
|
|
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
|
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.ScriptedAlarms;
|
|
|
|
/// <summary>
|
|
/// The alarm comment audit trail lives in its own append-only table
|
|
/// (<see cref="AlarmConditionStateSchema.CommentsTable"/>) keyed by alarm id, so the condition
|
|
/// row stays a fixed size and a transition's write — and therefore its replication payload —
|
|
/// does not grow with the alarm's history.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// The payload is measured where it costs: the bytes the LocalDb capture triggers write into
|
|
/// <c>__localdb_oplog</c> for one save, which is exactly what ships to the pair peer. Both
|
|
/// tables are registered for capture here, as they are on a replicating node.
|
|
/// </remarks>
|
|
public sealed class LocalDbAlarmCommentAuditTests : IDisposable
|
|
{
|
|
private static readonly DateTime T0 = new(2026, 06, 10, 12, 00, 00, DateTimeKind.Utc);
|
|
|
|
private readonly string _dbPath =
|
|
Path.Combine(Path.GetTempPath(), $"otopcua-alarm-comments-{Guid.NewGuid():N}.db");
|
|
|
|
private readonly ServiceProvider _provider;
|
|
private readonly ILocalDb _db;
|
|
|
|
public LocalDbAlarmCommentAuditTests()
|
|
{
|
|
var configuration = new ConfigurationBuilder()
|
|
.AddInMemoryCollection(new Dictionary<string, string?> { ["LocalDb:Path"] = _dbPath })
|
|
.Build();
|
|
|
|
_provider = new ServiceCollection()
|
|
.AddZbLocalDb(configuration, db =>
|
|
{
|
|
using (var connection = db.CreateConnection())
|
|
{
|
|
AlarmConditionStateSchema.Apply(connection);
|
|
}
|
|
|
|
db.RegisterReplicated(AlarmConditionStateSchema.StateTable);
|
|
db.RegisterReplicated(AlarmConditionStateSchema.CommentsTable);
|
|
})
|
|
.BuildServiceProvider();
|
|
|
|
_db = _provider.GetRequiredService<ILocalDb>();
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
_provider.Dispose();
|
|
SqliteConnection.ClearAllPools();
|
|
foreach (var path in new[] { _dbPath, $"{_dbPath}-wal", $"{_dbPath}-shm" })
|
|
{
|
|
try { if (File.Exists(path)) File.Delete(path); } catch { /* best effort */ }
|
|
}
|
|
}
|
|
|
|
private LocalDbAlarmConditionStateStore NewStore()
|
|
=> new(_db, NullLogger<LocalDbAlarmConditionStateStore>.Instance);
|
|
|
|
private static CancellationToken Ct => CancellationToken.None;
|
|
|
|
private static ImmutableList<AlarmComment> Comments(int count, string prefix = "c")
|
|
{
|
|
var builder = ImmutableList.CreateBuilder<AlarmComment>();
|
|
for (var i = 0; i < count; i++)
|
|
builder.Add(new AlarmComment(T0.AddSeconds(i), "jane", "AddComment", $"{prefix}-{i:D6}"));
|
|
return builder.ToImmutable();
|
|
}
|
|
|
|
/// <summary>An unacknowledged state carrying <paramref name="comments"/>, ready to be acked.</summary>
|
|
private static AlarmConditionState Unacked(string alarmId, ImmutableList<AlarmComment> comments) =>
|
|
AlarmConditionState.Fresh(alarmId, T0) with
|
|
{
|
|
Active = AlarmActiveState.Active,
|
|
Acked = AlarmAckedState.Unacknowledged,
|
|
Comments = comments,
|
|
};
|
|
|
|
private async Task<long> OplogHighWaterAsync() =>
|
|
(await _db.QueryAsync("SELECT COALESCE(MAX(seq), 0) FROM __localdb_oplog", r => r.GetInt64(0), null, Ct))[0];
|
|
|
|
/// <summary>Bytes captured into the oplog above <paramref name="afterSeq"/> — the replication payload.</summary>
|
|
private async Task<long> OplogBytesAboveAsync(long afterSeq) =>
|
|
(await _db.QueryAsync(
|
|
"SELECT COALESCE(SUM(LENGTH(pk_json) + COALESCE(LENGTH(row_json), 0)), 0) FROM __localdb_oplog WHERE seq > @S",
|
|
r => r.GetInt64(0),
|
|
new { S = afterSeq },
|
|
Ct))[0];
|
|
|
|
private async Task<long> OplogEntriesAboveAsync(long afterSeq, string table) =>
|
|
(await _db.QueryAsync(
|
|
"SELECT COUNT(*) FROM __localdb_oplog WHERE seq > @S AND table_name = @T",
|
|
r => r.GetInt64(0),
|
|
new { S = afterSeq, T = table },
|
|
Ct))[0];
|
|
|
|
private async Task<long> ScalarAsync(string sql) =>
|
|
(await _db.QueryAsync(sql, r => r.GetInt64(0), null, Ct))[0];
|
|
|
|
/// <summary>
|
|
/// Seeds <paramref name="alarmId"/> with <paramref name="historyLength"/> comments, then
|
|
/// returns the oplog bytes one acknowledge transition (one new comment) costs.
|
|
/// </summary>
|
|
private async Task<long> AckTransitionBytesAsync(string alarmId, int historyLength, string ackComment)
|
|
{
|
|
var store = NewStore();
|
|
var seeded = Unacked(alarmId, Comments(historyLength));
|
|
await store.SaveAsync(seeded, Ct);
|
|
|
|
var before = await OplogHighWaterAsync();
|
|
var acked = Part9StateMachine.ApplyAcknowledge(seeded, "bob", ackComment, T0.AddDays(1)).State;
|
|
await store.SaveAsync(acked, Ct);
|
|
return await OplogBytesAboveAsync(before);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task TransitionPayload_IsConstant_WhetherTheAlarmCarries10OrTenThousandComments()
|
|
{
|
|
var small = await AckTransitionBytesAsync("alarm-small", 10, "ack");
|
|
var large = await AckTransitionBytesAsync("alarm-large", 10_000, "ack");
|
|
|
|
// Constant: the only difference allowed is the width of the new comment's ordinal ("10" vs
|
|
// "10000", in pk_json and row_json) — a handful of bytes, not O(n). The alarm ids are the
|
|
// same length on purpose.
|
|
small.ShouldBeGreaterThan(0);
|
|
Math.Abs(large - small).ShouldBeLessThanOrEqualTo(16,
|
|
$"one ack transition cost {small} bytes at 10 comments but {large} at 10,000");
|
|
large.ShouldBeLessThan(2_048);
|
|
|
|
// Positive control on the instrument: the seeded history really is large and really was
|
|
// captured — otherwise a "constant" result could just mean capture is off.
|
|
(await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment WHERE scripted_alarm_id = 'alarm-large'"))
|
|
.ShouldBe(10_001);
|
|
var historyBytes = await ScalarAsync(
|
|
"SELECT SUM(LENGTH(row_json)) FROM __localdb_oplog WHERE table_name = 'alarm_condition_comment'");
|
|
historyBytes.ShouldBeGreaterThan(1_000_000);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task TransitionPayload_PositiveControl_TheMeasurementSeesTheNewCommentsBytes()
|
|
{
|
|
// Proves the measurement above is not vacuous: the oplog delta of a transition does carry
|
|
// the appended comment, so a 4 KB comment costs at least 4 KB.
|
|
var plain = await AckTransitionBytesAsync("alarm-plain", 10, "ack");
|
|
var heavy = await AckTransitionBytesAsync("alarm-heavy", 10, new string('x', 4_096));
|
|
|
|
(heavy - plain).ShouldBeGreaterThanOrEqualTo(4_096 * 2 - 16,
|
|
"the comment text rides both the comment row and last_ack_comment on the condition row");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Transition_WritesOneConditionRowAndOnlyTheAppendedCommentRow()
|
|
{
|
|
var store = NewStore();
|
|
var seeded = Unacked("alarm-append", Comments(50));
|
|
await store.SaveAsync(seeded, Ct);
|
|
|
|
var before = await OplogHighWaterAsync();
|
|
var acked = Part9StateMachine.ApplyAcknowledge(seeded, "bob", "ack", T0.AddDays(1)).State;
|
|
await store.SaveAsync(acked, Ct);
|
|
|
|
(await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(1);
|
|
(await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(1);
|
|
|
|
var loaded = await store.LoadAsync("alarm-append", Ct);
|
|
loaded.ShouldNotBeNull();
|
|
loaded.Comments.ShouldBe(acked.Comments);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ConditionRow_StaysFixedSize_InlineColumnIsAlwaysEmpty()
|
|
{
|
|
var store = NewStore();
|
|
await store.SaveAsync(Unacked("alarm-fixed", Comments(500)), Ct);
|
|
|
|
(await _db.QueryAsync(
|
|
"SELECT comments_json FROM alarm_condition_state WHERE scripted_alarm_id = 'alarm-fixed'",
|
|
r => r.GetString(0), null, Ct))[0]
|
|
.ShouldBe("[]");
|
|
(await store.LoadAsync("alarm-fixed", Ct))!.Comments.Count.ShouldBe(500);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task UnchangedSave_WritesNoCommentRows()
|
|
{
|
|
var store = NewStore();
|
|
var state = Unacked("alarm-idle", Comments(20));
|
|
await store.SaveAsync(state, Ct);
|
|
|
|
var before = await OplogHighWaterAsync();
|
|
await store.SaveAsync(state with { LastTransitionUtc = T0.AddMinutes(1) }, Ct);
|
|
|
|
(await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(0);
|
|
(await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(1,
|
|
"positive control: the save itself was captured");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task LoadAll_ReturnsEachAlarmsOwnTrailInOrder()
|
|
{
|
|
var store = NewStore();
|
|
var a = Unacked("alarm-a", Comments(3, "a"));
|
|
var b = Unacked("alarm-b", Comments(5, "b"));
|
|
await store.SaveAsync(a, Ct);
|
|
await store.SaveAsync(b, Ct);
|
|
|
|
var all = (await store.LoadAllAsync(Ct)).ToDictionary(s => s.AlarmId);
|
|
|
|
all["alarm-a"].Comments.ShouldBe(a.Comments);
|
|
all["alarm-b"].Comments.ShouldBe(b.Comments);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Remove_DeletesTheAlarmsTrail_AndAReCreatedAlarmStartsEmpty()
|
|
{
|
|
var store = NewStore();
|
|
await store.SaveAsync(Unacked("alarm-gone", Comments(4)), Ct);
|
|
await store.SaveAsync(Unacked("alarm-kept", Comments(2)), Ct);
|
|
|
|
await store.RemoveAsync("alarm-gone", Ct);
|
|
|
|
(await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment WHERE scripted_alarm_id = 'alarm-gone'"))
|
|
.ShouldBe(0);
|
|
(await store.LoadAsync("alarm-kept", Ct))!.Comments.Count.ShouldBe(2, "positive control");
|
|
|
|
var recreated = AlarmConditionState.Fresh("alarm-gone", T0)
|
|
with { Comments = ImmutableList.Create(new AlarmComment(T0, "amy", "AddComment", "new life")) };
|
|
await store.SaveAsync(recreated, Ct);
|
|
(await store.LoadAsync("alarm-gone", Ct))!.Comments.ShouldBe(recreated.Comments);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CommentsAlreadyHeldAtAnOrdinal_AreNeverRewritten()
|
|
{
|
|
// Append-only: a save whose list disagrees with what is stored at an existing ordinal (the
|
|
// peer's copy of the same logical entry already arrived) leaves the stored entry alone.
|
|
var store = NewStore();
|
|
var original = Unacked("alarm-ao", Comments(3));
|
|
await store.SaveAsync(original, Ct);
|
|
|
|
var rewritten = original with { Comments = original.Comments.SetItem(1, new AlarmComment(T0, "eve", "AddComment", "forged")) };
|
|
await store.SaveAsync(rewritten, Ct);
|
|
|
|
// The stored entry at ordinal 1 is untouched; the unrecognised local entry is kept by
|
|
// appending it (never dropped), not by overwriting history.
|
|
var loaded = (await store.LoadAsync("alarm-ao", Ct))!.Comments;
|
|
loaded.Take(3).ShouldBe(original.Comments);
|
|
loaded.Count.ShouldBe(4);
|
|
loaded[3].Text.ShouldBe("forged");
|
|
}
|
|
|
|
// ---- back-compat + migration -----------------------------------------------------------
|
|
|
|
/// <summary>
|
|
/// The exact shape the pre-split store wrote into <c>comments_json</c>
|
|
/// (<c>JsonSerializerDefaults.Web</c>: camelCase, Utc timestamps with a trailing <c>Z</c>).
|
|
/// </summary>
|
|
private const string OldShapeCommentsJson =
|
|
"""[{"timestampUtc":"2026-06-10T10:00:00Z","user":"jane","kind":"Acknowledge","text":"checking pump"},{"timestampUtc":"2026-06-10T11:30:00.1234567Z","user":"bob","kind":"Confirm","text":"all clear"},{"timestampUtc":"2026-06-10T11:30:00.1234567Z","user":"system","kind":"AutoUnshelve","text":""}]""";
|
|
|
|
private static readonly ImmutableList<AlarmComment> OldShapeComments = ImmutableList.Create(
|
|
new AlarmComment(new DateTime(2026, 06, 10, 10, 00, 00, DateTimeKind.Utc), "jane", "Acknowledge", "checking pump"),
|
|
new AlarmComment(new DateTime(2026, 06, 10, 11, 30, 00, DateTimeKind.Utc).AddTicks(1_234_567), "bob", "Confirm", "all clear"),
|
|
new AlarmComment(new DateTime(2026, 06, 10, 11, 30, 00, DateTimeKind.Utc).AddTicks(1_234_567), "system", "AutoUnshelve", ""));
|
|
|
|
private async Task InsertOldShapeRowAsync(string alarmId, string commentsJson = OldShapeCommentsJson) =>
|
|
await _db.ExecuteAsync(
|
|
"""
|
|
INSERT INTO alarm_condition_state
|
|
(scripted_alarm_id, enabled_state, acked_state, confirmed_state, shelving_state,
|
|
shelving_expires_utc, last_ack_user, last_ack_comment, last_ack_utc,
|
|
last_confirm_user, last_confirm_comment, last_confirm_utc, comments_json, updated_at_utc)
|
|
VALUES
|
|
(@Id, 'Enabled', 'Acknowledged', 'Confirmed', 'Unshelved',
|
|
NULL, 'jane', 'checking pump', '2026-06-10T10:00:00.0000000Z',
|
|
'bob', 'all clear', '2026-06-10T11:30:00.1234567Z', @Json, '2026-06-10T11:30:00.1234567Z')
|
|
""",
|
|
new { Id = alarmId, Json = commentsJson },
|
|
Ct);
|
|
|
|
private static void ShouldBeTheOldShapeState(AlarmConditionState? loaded, string alarmId)
|
|
{
|
|
loaded.ShouldNotBeNull();
|
|
loaded.AlarmId.ShouldBe(alarmId);
|
|
loaded.Acked.ShouldBe(AlarmAckedState.Acknowledged);
|
|
loaded.Confirmed.ShouldBe(AlarmConfirmedState.Confirmed);
|
|
loaded.LastAckUser.ShouldBe("jane");
|
|
loaded.LastConfirmComment.ShouldBe("all clear");
|
|
loaded.Comments.ShouldBe(OldShapeComments);
|
|
loaded.Comments.ShouldAllBe(c => c.TimestampUtc.Kind == DateTimeKind.Utc);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task OldShapeRow_WithInlineComments_LoadsExactlyAsBefore()
|
|
{
|
|
await InsertOldShapeRowAsync("alarm-legacy");
|
|
|
|
ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-legacy", Ct), "alarm-legacy");
|
|
ShouldBeTheOldShapeState((await NewStore().LoadAllAsync(Ct)).Single(), "alarm-legacy");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Migration_CopiesInlineCommentsToTheAuditTable_AndNeverRewritesTheConditionRow()
|
|
{
|
|
await InsertOldShapeRowAsync("alarm-m1");
|
|
await InsertOldShapeRowAsync("alarm-m2");
|
|
await InsertOldShapeRowAsync("alarm-empty", "[]");
|
|
|
|
var before = await OplogHighWaterAsync();
|
|
AlarmConditionStateSchema.InlineCommentMigrationResult result;
|
|
using (var connection = _db.CreateConnection())
|
|
result = AlarmConditionStateSchema.MigrateInlineComments(connection);
|
|
|
|
result.RowsMoved.ShouldBe(2);
|
|
result.UnparseableAlarmIds.ShouldBeEmpty();
|
|
(await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment")).ShouldBe(6);
|
|
|
|
// Review m1: the migration must not touch the condition row. A full-row UPDATE re-stamps it
|
|
// with a fresh HLC and can revert a newer peer state (e.g. an ack) under LWW. The next save
|
|
// (the engine's boot save) resets the column instead.
|
|
(await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(0);
|
|
(await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(6,
|
|
"positive control: the copied entries were captured");
|
|
|
|
ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-m1", Ct), "alarm-m1");
|
|
ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-m2", Ct), "alarm-m2");
|
|
(await NewStore().LoadAsync("alarm-empty", Ct))!.Comments.ShouldBeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Migration_IsIdempotent_ASecondRunChangesNothing()
|
|
{
|
|
await InsertOldShapeRowAsync("alarm-idem");
|
|
using (var connection = _db.CreateConnection())
|
|
AlarmConditionStateSchema.MigrateInlineComments(connection).RowsMoved.ShouldBe(1, "positive control");
|
|
|
|
var before = await OplogHighWaterAsync();
|
|
using (var connection = _db.CreateConnection())
|
|
AlarmConditionStateSchema.MigrateInlineComments(connection).RowsMoved.ShouldBe(0);
|
|
|
|
(await OplogHighWaterAsync()).ShouldBe(before, "a re-run must write nothing");
|
|
(await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment")).ShouldBe(3);
|
|
ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-idem", Ct), "alarm-idem");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Migration_OverAPartiallyMigratedRow_FillsOnlyTheMissingOrdinals()
|
|
{
|
|
// Only ordinal 0 made it across (e.g. a restored backup): the run fills 1 and 2, keeps 0,
|
|
// duplicates nothing.
|
|
await InsertOldShapeRowAsync("alarm-partial");
|
|
await _db.ExecuteAsync(
|
|
AlarmConditionStateSchema.InsertCommentSql,
|
|
new
|
|
{
|
|
Id = "alarm-partial", Seq = 0,
|
|
Ts = AlarmConditionStateSchema.FormatTimestamp(OldShapeComments[0].TimestampUtc),
|
|
User = OldShapeComments[0].User, Kind = OldShapeComments[0].Kind, Text = OldShapeComments[0].Text,
|
|
},
|
|
Ct);
|
|
|
|
using (var connection = _db.CreateConnection())
|
|
AlarmConditionStateSchema.MigrateInlineComments(connection).RowsMoved.ShouldBe(1);
|
|
|
|
(await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment")).ShouldBe(3);
|
|
ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-partial", Ct), "alarm-partial");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Migration_ReportsAnUnparseableInlineTrail_AndLeavesItAlone()
|
|
{
|
|
// Review m2: an unreadable trail must surface (OnReady logs a Warning per id), not be
|
|
// skipped silently on every boot.
|
|
await InsertOldShapeRowAsync("alarm-bad", "{not json");
|
|
await InsertOldShapeRowAsync("alarm-good");
|
|
|
|
AlarmConditionStateSchema.InlineCommentMigrationResult result;
|
|
using (var connection = _db.CreateConnection())
|
|
result = AlarmConditionStateSchema.MigrateInlineComments(connection);
|
|
|
|
result.UnparseableAlarmIds.ShouldBe(["alarm-bad"]);
|
|
result.RowsMoved.ShouldBe(1, "positive control: the good row still moved");
|
|
(await _db.QueryAsync(
|
|
"SELECT comments_json FROM alarm_condition_state WHERE scripted_alarm_id = 'alarm-bad'",
|
|
r => r.GetString(0), null, Ct))[0]
|
|
.ShouldBe("{not json");
|
|
}
|
|
|
|
// ---- divergence / failover (review I-1) -------------------------------------------------
|
|
|
|
/// <summary>Captures log entries so a divergence Warning can be asserted, not inferred.</summary>
|
|
private sealed class CapturingLogger : Microsoft.Extensions.Logging.ILogger<LocalDbAlarmConditionStateStore>
|
|
{
|
|
public List<(Microsoft.Extensions.Logging.LogLevel Level, string Message)> Entries { get; } = [];
|
|
|
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
|
|
|
public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true;
|
|
|
|
public void Log<TState>(
|
|
Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId,
|
|
TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
|
{
|
|
lock (Entries) Entries.Add((logLevel, formatter(state, exception)));
|
|
}
|
|
|
|
public int Warnings => Entries.Count(e => e.Level == Microsoft.Extensions.Logging.LogLevel.Warning);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Failover_TheSurvivorsNextEntry_IsPersisted_AfterItMissedACommandThePeerApplied()
|
|
{
|
|
// The I-1 sequence on node B's database:
|
|
// 1. a shared trail of 5 entries;
|
|
// 2. an AddComment reaches only node A (B's command delivery failed once) and A's row at
|
|
// ordinal 5 replicates into B's table — B's IN-MEMORY trail still has 5 entries;
|
|
// 3. A dies and the operator acks on B: B's in-memory Ack sits at index 5, colliding with
|
|
// A's entry at ordinal 5.
|
|
// The Ack must be persisted (and say so), not silently treated as "already stored".
|
|
var shared = Unacked("alarm-fo", Comments(5));
|
|
var a = NewStore();
|
|
await a.SaveAsync(shared, Ct);
|
|
var onlyOnA = Part9StateMachine.ApplyAddComment(shared, "jane", "seen on A only", T0.AddHours(1)).State;
|
|
await a.SaveAsync(onlyOnA, Ct);
|
|
|
|
var log = new CapturingLogger();
|
|
var b = new LocalDbAlarmConditionStateStore(_db, log);
|
|
var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "acked on B", T0.AddHours(2)).State;
|
|
await b.SaveAsync(ackOnB, Ct);
|
|
|
|
var loaded = (await NewStore().LoadAsync("alarm-fo", Ct))!;
|
|
loaded.Comments.Count.ShouldBe(7);
|
|
loaded.Comments[5].ShouldBe(onlyOnA.Comments[5]);
|
|
loaded.Comments[6].ShouldBe(ackOnB.Comments[5]);
|
|
log.Warnings.ShouldBe(1, "a divergence is never silent");
|
|
|
|
// Re-saving with no new entry must not append the Ack a second time (its own earlier
|
|
// divergence-append shifted it off its in-memory index).
|
|
var before = await OplogHighWaterAsync();
|
|
await b.SaveAsync(ackOnB with { LastTransitionUtc = T0.AddHours(3) }, Ct);
|
|
(await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(0);
|
|
(await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(1,
|
|
"positive control: the re-save itself was captured");
|
|
|
|
// And B's following entry (still one behind in memory) is persisted too, exactly once.
|
|
var confirmOnB = Part9StateMachine.ApplyAddComment(ackOnB, "bob", "follow-up on B", T0.AddHours(4)).State;
|
|
await b.SaveAsync(confirmOnB, Ct);
|
|
await b.SaveAsync(confirmOnB, Ct);
|
|
var after = (await NewStore().LoadAsync("alarm-fo", Ct))!;
|
|
after.Comments.Count.ShouldBe(8);
|
|
after.Comments[7].ShouldBe(confirmOnB.Comments[6]);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PeersCopyOfTheSameCommand_IsNotAppendedAgain_AndLogsNothing()
|
|
{
|
|
// The positive control for the divergence path: in normal operation both nodes apply every
|
|
// command, each with its own clock. When the peer's copy is already at the ordinal, the
|
|
// local copy is the same entry and must NOT be appended — otherwise the fix would double
|
|
// every entry on a healthy pair.
|
|
var shared = Unacked("alarm-dual", Comments(5));
|
|
await NewStore().SaveAsync(shared, Ct);
|
|
var ackOnA = Part9StateMachine.ApplyAcknowledge(shared, "bob", "on it", T0.AddHours(1)).State;
|
|
await NewStore().SaveAsync(ackOnA, Ct);
|
|
|
|
var log = new CapturingLogger();
|
|
var b = new LocalDbAlarmConditionStateStore(_db, log);
|
|
var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "on it", T0.AddHours(1).AddMilliseconds(40)).State;
|
|
var before = await OplogHighWaterAsync();
|
|
await b.SaveAsync(ackOnB, Ct);
|
|
|
|
(await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(0);
|
|
(await NewStore().LoadAsync("alarm-dual", Ct))!.Comments.Count.ShouldBe(6);
|
|
log.Warnings.ShouldBe(0);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ARepeatedIdenticalComment_IsKept_AsItsOwnEntry()
|
|
{
|
|
// Matching ignores the timestamp only within the clock-skew window; an operator repeating the
|
|
// same text later is a new entry, not a duplicate of the earlier one.
|
|
var store = NewStore();
|
|
var s = Unacked("alarm-repeat", Comments(2));
|
|
s = Part9StateMachine.ApplyAddComment(s, "amy", "still high", T0.AddHours(1)).State;
|
|
await store.SaveAsync(s, Ct);
|
|
s = Part9StateMachine.ApplyAddComment(s, "amy", "still high", T0.AddHours(2)).State;
|
|
await store.SaveAsync(s, Ct);
|
|
|
|
(await NewStore().LoadAsync("alarm-repeat", Ct))!.Comments.ShouldBe(s.Comments);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SavingALoadedOldShapeState_MovesItsCommentsAndAppendsTheNewOne()
|
|
{
|
|
// The lazy path: an old-shape row the boot migration never saw (it was written before the
|
|
// first save on this build) still converges on the first save.
|
|
await InsertOldShapeRowAsync("alarm-lazy");
|
|
var store = NewStore();
|
|
var loaded = await store.LoadAsync("alarm-lazy", Ct);
|
|
|
|
var next = Part9StateMachine.ApplyAddComment(loaded!, "amy", "follow-up", T0.AddDays(2)).State;
|
|
await store.SaveAsync(next, Ct);
|
|
|
|
(await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment WHERE scripted_alarm_id = 'alarm-lazy'"))
|
|
.ShouldBe(4);
|
|
(await _db.QueryAsync(
|
|
"SELECT comments_json FROM alarm_condition_state WHERE scripted_alarm_id = 'alarm-lazy'",
|
|
r => r.GetString(0), null, Ct))[0]
|
|
.ShouldBe("[]");
|
|
(await NewStore().LoadAsync("alarm-lazy", Ct))!.Comments.ShouldBe(next.Comments);
|
|
}
|
|
}
|