using System.Collections.Immutable; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; using Xunit; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.ScriptedAlarms; /// /// The alarm comment audit trail lives in its own append-only table /// () keyed by alarm id, so the condition /// row stays a fixed size and a transition's write — and therefore its replication payload — /// does not grow with the alarm's history. /// /// /// The payload is measured where it costs: the bytes the LocalDb capture triggers write into /// __localdb_oplog for one save, which is exactly what ships to the pair peer. Both /// tables are registered for capture here, as they are on a replicating node. /// public sealed class LocalDbAlarmCommentAuditTests : IDisposable { private static readonly DateTime T0 = new(2026, 06, 10, 12, 00, 00, DateTimeKind.Utc); private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"otopcua-alarm-comments-{Guid.NewGuid():N}.db"); private readonly ServiceProvider _provider; private readonly ILocalDb _db; public LocalDbAlarmCommentAuditTests() { var configuration = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary { ["LocalDb:Path"] = _dbPath }) .Build(); _provider = new ServiceCollection() .AddZbLocalDb(configuration, db => { using (var connection = db.CreateConnection()) { AlarmConditionStateSchema.Apply(connection); } db.RegisterReplicated(AlarmConditionStateSchema.StateTable); db.RegisterReplicated(AlarmConditionStateSchema.CommentsTable); }) .BuildServiceProvider(); _db = _provider.GetRequiredService(); } public void Dispose() { _provider.Dispose(); SqliteConnection.ClearAllPools(); foreach (var path in new[] { _dbPath, $"{_dbPath}-wal", $"{_dbPath}-shm" }) { try { if (File.Exists(path)) File.Delete(path); } catch { /* best effort */ } } } private LocalDbAlarmConditionStateStore NewStore() => new(_db, NullLogger.Instance); private static CancellationToken Ct => CancellationToken.None; private static ImmutableList Comments(int count, string prefix = "c") { var builder = ImmutableList.CreateBuilder(); for (var i = 0; i < count; i++) builder.Add(new AlarmComment(T0.AddSeconds(i), "jane", "AddComment", $"{prefix}-{i:D6}")); return builder.ToImmutable(); } /// An unacknowledged state carrying , ready to be acked. private static AlarmConditionState Unacked(string alarmId, ImmutableList comments) => AlarmConditionState.Fresh(alarmId, T0) with { Active = AlarmActiveState.Active, Acked = AlarmAckedState.Unacknowledged, Comments = comments, }; private async Task OplogHighWaterAsync() => (await _db.QueryAsync("SELECT COALESCE(MAX(seq), 0) FROM __localdb_oplog", r => r.GetInt64(0), null, Ct))[0]; /// Bytes captured into the oplog above — the replication payload. private async Task OplogBytesAboveAsync(long afterSeq) => (await _db.QueryAsync( "SELECT COALESCE(SUM(LENGTH(pk_json) + COALESCE(LENGTH(row_json), 0)), 0) FROM __localdb_oplog WHERE seq > @S", r => r.GetInt64(0), new { S = afterSeq }, Ct))[0]; private async Task OplogEntriesAboveAsync(long afterSeq, string table) => (await _db.QueryAsync( "SELECT COUNT(*) FROM __localdb_oplog WHERE seq > @S AND table_name = @T", r => r.GetInt64(0), new { S = afterSeq, T = table }, Ct))[0]; private async Task ScalarAsync(string sql) => (await _db.QueryAsync(sql, r => r.GetInt64(0), null, Ct))[0]; /// /// Seeds with comments, then /// returns the oplog bytes one acknowledge transition (one new comment) costs. /// private async Task AckTransitionBytesAsync(string alarmId, int historyLength, string ackComment) { var store = NewStore(); var seeded = Unacked(alarmId, Comments(historyLength)); await store.SaveAsync(seeded, Ct); var before = await OplogHighWaterAsync(); var acked = Part9StateMachine.ApplyAcknowledge(seeded, "bob", ackComment, T0.AddDays(1)).State; await store.SaveAsync(acked, Ct); return await OplogBytesAboveAsync(before); } [Fact] public async Task TransitionPayload_IsConstant_WhetherTheAlarmCarries10OrTenThousandComments() { var small = await AckTransitionBytesAsync("alarm-small", 10, "ack"); var large = await AckTransitionBytesAsync("alarm-large", 10_000, "ack"); // Constant: the only difference allowed is the width of the new comment's ordinal ("10" vs // "10000", in pk_json and row_json) — a handful of bytes, not O(n). The alarm ids are the // same length on purpose. small.ShouldBeGreaterThan(0); Math.Abs(large - small).ShouldBeLessThanOrEqualTo(16, $"one ack transition cost {small} bytes at 10 comments but {large} at 10,000"); large.ShouldBeLessThan(2_048); // Positive control on the instrument: the seeded history really is large and really was // captured — otherwise a "constant" result could just mean capture is off. (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment WHERE scripted_alarm_id = 'alarm-large'")) .ShouldBe(10_001); var historyBytes = await ScalarAsync( "SELECT SUM(LENGTH(row_json)) FROM __localdb_oplog WHERE table_name = 'alarm_condition_comment'"); historyBytes.ShouldBeGreaterThan(1_000_000); } [Fact] public async Task TransitionPayload_PositiveControl_TheMeasurementSeesTheNewCommentsBytes() { // Proves the measurement above is not vacuous: the oplog delta of a transition does carry // the appended comment, so a 4 KB comment costs at least 4 KB. var plain = await AckTransitionBytesAsync("alarm-plain", 10, "ack"); var heavy = await AckTransitionBytesAsync("alarm-heavy", 10, new string('x', 4_096)); (heavy - plain).ShouldBeGreaterThanOrEqualTo(4_096 * 2 - 16, "the comment text rides both the comment row and last_ack_comment on the condition row"); } [Fact] public async Task Transition_WritesOneConditionRowAndOnlyTheAppendedCommentRow() { var store = NewStore(); var seeded = Unacked("alarm-append", Comments(50)); await store.SaveAsync(seeded, Ct); var before = await OplogHighWaterAsync(); var acked = Part9StateMachine.ApplyAcknowledge(seeded, "bob", "ack", T0.AddDays(1)).State; await store.SaveAsync(acked, Ct); (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(1); (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(1); var loaded = await store.LoadAsync("alarm-append", Ct); loaded.ShouldNotBeNull(); loaded.Comments.ShouldBe(acked.Comments); } [Fact] public async Task ConditionRow_StaysFixedSize_InlineColumnIsAlwaysEmpty() { var store = NewStore(); await store.SaveAsync(Unacked("alarm-fixed", Comments(500)), Ct); (await _db.QueryAsync( "SELECT comments_json FROM alarm_condition_state WHERE scripted_alarm_id = 'alarm-fixed'", r => r.GetString(0), null, Ct))[0] .ShouldBe("[]"); (await store.LoadAsync("alarm-fixed", Ct))!.Comments.Count.ShouldBe(500); } [Fact] public async Task UnchangedSave_WritesNoCommentRows() { var store = NewStore(); var state = Unacked("alarm-idle", Comments(20)); await store.SaveAsync(state, Ct); var before = await OplogHighWaterAsync(); await store.SaveAsync(state with { LastTransitionUtc = T0.AddMinutes(1) }, Ct); (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(0); (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(1, "positive control: the save itself was captured"); } [Fact] public async Task LoadAll_ReturnsEachAlarmsOwnTrailInOrder() { var store = NewStore(); var a = Unacked("alarm-a", Comments(3, "a")); var b = Unacked("alarm-b", Comments(5, "b")); await store.SaveAsync(a, Ct); await store.SaveAsync(b, Ct); var all = (await store.LoadAllAsync(Ct)).ToDictionary(s => s.AlarmId); all["alarm-a"].Comments.ShouldBe(a.Comments); all["alarm-b"].Comments.ShouldBe(b.Comments); } [Fact] public async Task Remove_DeletesTheAlarmsTrail_AndAReCreatedAlarmStartsEmpty() { var store = NewStore(); await store.SaveAsync(Unacked("alarm-gone", Comments(4)), Ct); await store.SaveAsync(Unacked("alarm-kept", Comments(2)), Ct); await store.RemoveAsync("alarm-gone", Ct); (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment WHERE scripted_alarm_id = 'alarm-gone'")) .ShouldBe(0); (await store.LoadAsync("alarm-kept", Ct))!.Comments.Count.ShouldBe(2, "positive control"); var recreated = AlarmConditionState.Fresh("alarm-gone", T0) with { Comments = ImmutableList.Create(new AlarmComment(T0, "amy", "AddComment", "new life")) }; await store.SaveAsync(recreated, Ct); (await store.LoadAsync("alarm-gone", Ct))!.Comments.ShouldBe(recreated.Comments); } [Fact] public async Task CommentsAlreadyHeldAtAnOrdinal_AreNeverRewritten() { // Append-only: a save whose list disagrees with what is stored at an existing ordinal (the // peer's copy of the same logical entry already arrived) leaves the stored entry alone. var store = NewStore(); var original = Unacked("alarm-ao", Comments(3)); await store.SaveAsync(original, Ct); var rewritten = original with { Comments = original.Comments.SetItem(1, new AlarmComment(T0, "eve", "AddComment", "forged")) }; await store.SaveAsync(rewritten, Ct); // The stored entry at ordinal 1 is untouched; the unrecognised local entry is kept by // appending it (never dropped), not by overwriting history. var loaded = (await store.LoadAsync("alarm-ao", Ct))!.Comments; loaded.Take(3).ShouldBe(original.Comments); loaded.Count.ShouldBe(4); loaded[3].Text.ShouldBe("forged"); } // ---- back-compat + migration ----------------------------------------------------------- /// /// The exact shape the pre-split store wrote into comments_json /// (JsonSerializerDefaults.Web: camelCase, Utc timestamps with a trailing Z). /// private const string OldShapeCommentsJson = """[{"timestampUtc":"2026-06-10T10:00:00Z","user":"jane","kind":"Acknowledge","text":"checking pump"},{"timestampUtc":"2026-06-10T11:30:00.1234567Z","user":"bob","kind":"Confirm","text":"all clear"},{"timestampUtc":"2026-06-10T11:30:00.1234567Z","user":"system","kind":"AutoUnshelve","text":""}]"""; private static readonly ImmutableList OldShapeComments = ImmutableList.Create( new AlarmComment(new DateTime(2026, 06, 10, 10, 00, 00, DateTimeKind.Utc), "jane", "Acknowledge", "checking pump"), new AlarmComment(new DateTime(2026, 06, 10, 11, 30, 00, DateTimeKind.Utc).AddTicks(1_234_567), "bob", "Confirm", "all clear"), new AlarmComment(new DateTime(2026, 06, 10, 11, 30, 00, DateTimeKind.Utc).AddTicks(1_234_567), "system", "AutoUnshelve", "")); private async Task InsertOldShapeRowAsync(string alarmId, string commentsJson = OldShapeCommentsJson) => await _db.ExecuteAsync( """ INSERT INTO alarm_condition_state (scripted_alarm_id, enabled_state, acked_state, confirmed_state, shelving_state, shelving_expires_utc, last_ack_user, last_ack_comment, last_ack_utc, last_confirm_user, last_confirm_comment, last_confirm_utc, comments_json, updated_at_utc) VALUES (@Id, 'Enabled', 'Acknowledged', 'Confirmed', 'Unshelved', NULL, 'jane', 'checking pump', '2026-06-10T10:00:00.0000000Z', 'bob', 'all clear', '2026-06-10T11:30:00.1234567Z', @Json, '2026-06-10T11:30:00.1234567Z') """, new { Id = alarmId, Json = commentsJson }, Ct); private static void ShouldBeTheOldShapeState(AlarmConditionState? loaded, string alarmId) { loaded.ShouldNotBeNull(); loaded.AlarmId.ShouldBe(alarmId); loaded.Acked.ShouldBe(AlarmAckedState.Acknowledged); loaded.Confirmed.ShouldBe(AlarmConfirmedState.Confirmed); loaded.LastAckUser.ShouldBe("jane"); loaded.LastConfirmComment.ShouldBe("all clear"); loaded.Comments.ShouldBe(OldShapeComments); loaded.Comments.ShouldAllBe(c => c.TimestampUtc.Kind == DateTimeKind.Utc); } [Fact] public async Task OldShapeRow_WithInlineComments_LoadsExactlyAsBefore() { await InsertOldShapeRowAsync("alarm-legacy"); ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-legacy", Ct), "alarm-legacy"); ShouldBeTheOldShapeState((await NewStore().LoadAllAsync(Ct)).Single(), "alarm-legacy"); } [Fact] public async Task Migration_CopiesInlineCommentsToTheAuditTable_AndNeverRewritesTheConditionRow() { await InsertOldShapeRowAsync("alarm-m1"); await InsertOldShapeRowAsync("alarm-m2"); await InsertOldShapeRowAsync("alarm-empty", "[]"); var before = await OplogHighWaterAsync(); AlarmConditionStateSchema.InlineCommentMigrationResult result; using (var connection = _db.CreateConnection()) result = AlarmConditionStateSchema.MigrateInlineComments(connection); result.RowsMoved.ShouldBe(2); result.UnparseableAlarmIds.ShouldBeEmpty(); (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment")).ShouldBe(6); // Review m1: the migration must not touch the condition row. A full-row UPDATE re-stamps it // with a fresh HLC and can revert a newer peer state (e.g. an ack) under LWW. The next save // (the engine's boot save) resets the column instead. (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(0); (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(6, "positive control: the copied entries were captured"); ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-m1", Ct), "alarm-m1"); ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-m2", Ct), "alarm-m2"); (await NewStore().LoadAsync("alarm-empty", Ct))!.Comments.ShouldBeEmpty(); } [Fact] public async Task Migration_IsIdempotent_ASecondRunChangesNothing() { await InsertOldShapeRowAsync("alarm-idem"); using (var connection = _db.CreateConnection()) AlarmConditionStateSchema.MigrateInlineComments(connection).RowsMoved.ShouldBe(1, "positive control"); var before = await OplogHighWaterAsync(); using (var connection = _db.CreateConnection()) AlarmConditionStateSchema.MigrateInlineComments(connection).RowsMoved.ShouldBe(0); (await OplogHighWaterAsync()).ShouldBe(before, "a re-run must write nothing"); (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment")).ShouldBe(3); ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-idem", Ct), "alarm-idem"); } [Fact] public async Task Migration_OverAPartiallyMigratedRow_FillsOnlyTheMissingOrdinals() { // Only ordinal 0 made it across (e.g. a restored backup): the run fills 1 and 2, keeps 0, // duplicates nothing. await InsertOldShapeRowAsync("alarm-partial"); await _db.ExecuteAsync( AlarmConditionStateSchema.InsertCommentSql, new { Id = "alarm-partial", Seq = 0, Ts = AlarmConditionStateSchema.FormatTimestamp(OldShapeComments[0].TimestampUtc), User = OldShapeComments[0].User, Kind = OldShapeComments[0].Kind, Text = OldShapeComments[0].Text, }, Ct); using (var connection = _db.CreateConnection()) AlarmConditionStateSchema.MigrateInlineComments(connection).RowsMoved.ShouldBe(1); (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment")).ShouldBe(3); ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-partial", Ct), "alarm-partial"); } [Fact] public async Task Migration_ReportsAnUnparseableInlineTrail_AndLeavesItAlone() { // Review m2: an unreadable trail must surface (OnReady logs a Warning per id), not be // skipped silently on every boot. await InsertOldShapeRowAsync("alarm-bad", "{not json"); await InsertOldShapeRowAsync("alarm-good"); AlarmConditionStateSchema.InlineCommentMigrationResult result; using (var connection = _db.CreateConnection()) result = AlarmConditionStateSchema.MigrateInlineComments(connection); result.UnparseableAlarmIds.ShouldBe(["alarm-bad"]); result.RowsMoved.ShouldBe(1, "positive control: the good row still moved"); (await _db.QueryAsync( "SELECT comments_json FROM alarm_condition_state WHERE scripted_alarm_id = 'alarm-bad'", r => r.GetString(0), null, Ct))[0] .ShouldBe("{not json"); } // ---- divergence / failover (review I-1) ------------------------------------------------- /// Captures log entries so a divergence Warning can be asserted, not inferred. private sealed class CapturingLogger : Microsoft.Extensions.Logging.ILogger { public List<(Microsoft.Extensions.Logging.LogLevel Level, string Message)> Entries { get; } = []; public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true; public void Log( Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId, TState state, Exception? exception, Func formatter) { lock (Entries) Entries.Add((logLevel, formatter(state, exception))); } public int Warnings => Entries.Count(e => e.Level == Microsoft.Extensions.Logging.LogLevel.Warning); } [Fact] public async Task Failover_TheSurvivorsNextEntry_IsPersisted_AfterItMissedACommandThePeerApplied() { // The I-1 sequence on node B's database: // 1. a shared trail of 5 entries; // 2. an AddComment reaches only node A (B's command delivery failed once) and A's row at // ordinal 5 replicates into B's table — B's IN-MEMORY trail still has 5 entries; // 3. A dies and the operator acks on B: B's in-memory Ack sits at index 5, colliding with // A's entry at ordinal 5. // The Ack must be persisted (and say so), not silently treated as "already stored". var shared = Unacked("alarm-fo", Comments(5)); var a = NewStore(); await a.SaveAsync(shared, Ct); var onlyOnA = Part9StateMachine.ApplyAddComment(shared, "jane", "seen on A only", T0.AddHours(1)).State; await a.SaveAsync(onlyOnA, Ct); var log = new CapturingLogger(); var b = new LocalDbAlarmConditionStateStore(_db, log); var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "acked on B", T0.AddHours(2)).State; await b.SaveAsync(ackOnB, Ct); var loaded = (await NewStore().LoadAsync("alarm-fo", Ct))!; loaded.Comments.Count.ShouldBe(7); loaded.Comments[5].ShouldBe(onlyOnA.Comments[5]); loaded.Comments[6].ShouldBe(ackOnB.Comments[5]); log.Warnings.ShouldBe(1, "a divergence is never silent"); // Re-saving with no new entry must not append the Ack a second time (its own earlier // divergence-append shifted it off its in-memory index). var before = await OplogHighWaterAsync(); await b.SaveAsync(ackOnB with { LastTransitionUtc = T0.AddHours(3) }, Ct); (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(0); (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(1, "positive control: the re-save itself was captured"); // And B's following entry (still one behind in memory) is persisted too, exactly once. var confirmOnB = Part9StateMachine.ApplyAddComment(ackOnB, "bob", "follow-up on B", T0.AddHours(4)).State; await b.SaveAsync(confirmOnB, Ct); await b.SaveAsync(confirmOnB, Ct); var after = (await NewStore().LoadAsync("alarm-fo", Ct))!; after.Comments.Count.ShouldBe(8); after.Comments[7].ShouldBe(confirmOnB.Comments[6]); } [Fact] public async Task PeersCopyOfTheSameCommand_IsNotAppendedAgain_AndLogsNothing() { // The positive control for the divergence path: in normal operation both nodes apply every // command, each with its own clock. When the peer's copy is already at the ordinal, the // local copy is the same entry and must NOT be appended — otherwise the fix would double // every entry on a healthy pair. var shared = Unacked("alarm-dual", Comments(5)); await NewStore().SaveAsync(shared, Ct); var ackOnA = Part9StateMachine.ApplyAcknowledge(shared, "bob", "on it", T0.AddHours(1)).State; await NewStore().SaveAsync(ackOnA, Ct); var log = new CapturingLogger(); var b = new LocalDbAlarmConditionStateStore(_db, log); var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "on it", T0.AddHours(1).AddMilliseconds(40)).State; var before = await OplogHighWaterAsync(); await b.SaveAsync(ackOnB, Ct); (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(0); (await NewStore().LoadAsync("alarm-dual", Ct))!.Comments.Count.ShouldBe(6); log.Warnings.ShouldBe(0); } [Fact] public async Task ARepeatedIdenticalComment_IsKept_AsItsOwnEntry() { // Matching ignores the timestamp only within the clock-skew window; an operator repeating the // same text later is a new entry, not a duplicate of the earlier one. var store = NewStore(); var s = Unacked("alarm-repeat", Comments(2)); s = Part9StateMachine.ApplyAddComment(s, "amy", "still high", T0.AddHours(1)).State; await store.SaveAsync(s, Ct); s = Part9StateMachine.ApplyAddComment(s, "amy", "still high", T0.AddHours(2)).State; await store.SaveAsync(s, Ct); (await NewStore().LoadAsync("alarm-repeat", Ct))!.Comments.ShouldBe(s.Comments); } [Fact] public async Task SavingALoadedOldShapeState_MovesItsCommentsAndAppendsTheNewOne() { // The lazy path: an old-shape row the boot migration never saw (it was written before the // first save on this build) still converges on the first save. await InsertOldShapeRowAsync("alarm-lazy"); var store = NewStore(); var loaded = await store.LoadAsync("alarm-lazy", Ct); var next = Part9StateMachine.ApplyAddComment(loaded!, "amy", "follow-up", T0.AddDays(2)).State; await store.SaveAsync(next, Ct); (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment WHERE scripted_alarm_id = 'alarm-lazy'")) .ShouldBe(4); (await _db.QueryAsync( "SELECT comments_json FROM alarm_condition_state WHERE scripted_alarm_id = 'alarm-lazy'", r => r.GetString(0), null, Ct))[0] .ShouldBe("[]"); (await NewStore().LoadAsync("alarm-lazy", Ct))!.Comments.ShouldBe(next.Comments); } }