Files
lmxopcua/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs
T
Joseph Doherty 6cbf763719 fix(alarms): engine cache prune + bounded dispose, source Warning, cached dead-letter depth, script-log rate limit (E3)
- R05:1/12 ScriptedAlarmEngine: _valueCache is cleared in LoadAsync (it is
  re-seeded for the referenced paths) so retired tags no longer accrete across
  deploys; OnUpstreamChange no longer writes the cache after Dispose.
- R05:3 ScriptedAlarmEngine.Dispose drains in-flight work for at most 5 s
  (DisposeDrainTimeout), then cancels it through a shutdown token every
  DrainDirtyAsync/ShelvingCheckAsync now runs on, logs a Warning and gives
  one second of grace — a stalled IAlarmStateStore.SaveAsync can no longer
  hang host shutdown. A save that completes inside the bound is still drained
  (positive control test).
- R05:6 ScriptedAlarmSource logs a throwing OnAlarmEvent subscriber at
  Warning (naming the alarm) instead of swallowing it.
- R05:7 LocalDbStoreAndForwardSink keeps a cached dead-letter counter in step
  with every dead-letter/revive/purge path and resyncs it with the queued
  counter on the same cadence; GetStatus no longer runs a COUNT(*) scan.
- R05:9 ScriptLogRateLimiter (20 per source per minute, fixed window, one
  summary entry naming the dropped count when the window rolls) behind
  ScriptLogTopicSink (key = script|vtag|alarm) and ScriptLogCompanionSink
  (key = script name).

Refs #734

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 10:17:30 -04:00

160 lines
6.6 KiB
C#

using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests;
/// <summary>
/// Review R05:1 / R05:12 / R05:3 — the engine's value cache is pruned on every reload (it used to keep
/// every path ever referenced, forever), is no longer written after Dispose, and Dispose's in-flight
/// drain is bounded (a hung store no longer hangs host shutdown).
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptedAlarmEngineLifecycleTests
{
private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);
private static ScriptedAlarmEngine Build(FakeUpstream up, IAlarmStateStore? store = null)
{
var logger = new LoggerConfiguration().CreateLogger();
return new ScriptedAlarmEngine(up, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
}
[Fact]
public async Task Reload_prunes_value_cache_entries_for_paths_the_new_alarm_set_no_longer_references()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
up.Set("Pressure", 5);
using var eng = Build(up);
var ct = TestContext.Current.CancellationToken;
await eng.LoadAsync(
[Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100 || (int)ctx.GetTag("Pressure").Value > 10;""")], ct);
eng.ValueCacheCountForTest.ShouldBe(2);
// The redeploy drops Pressure from every predicate: its entry must go with it.
await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], ct);
eng.ValueCacheCountForTest.ShouldBe(1);
eng.ValueCacheContainsForTest("Pressure").ShouldBeFalse();
eng.ValueCacheContainsForTest("Temp").ShouldBeTrue();
}
[Fact]
public async Task A_push_after_Dispose_does_not_write_the_value_cache()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
var eng = Build(up);
await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
eng.Dispose();
eng.ValueCacheCountForTest.ShouldBe(1);
eng.OnUpstreamChange("Late", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow));
eng.ValueCacheContainsForTest("Late").ShouldBeFalse("a disposed engine must not grow its cache");
}
[Fact]
public async Task Dispose_gives_up_on_a_store_that_never_returns_after_the_drain_bound_and_cancels_it()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
var store = new NeverReturningSaveStore();
var eng = Build(up, store);
eng.DisposeDrainTimeout = TimeSpan.FromMilliseconds(300);
await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
store.BlockNext = true;
up.Push("Temp", 150); // Activated → SaveAsync → blocks forever (until cancelled)
await WaitForAsync(() => store.SaveInProgress);
var sw = System.Diagnostics.Stopwatch.StartNew();
eng.Dispose();
sw.Stop();
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the drain is bounded; a hung store cannot hang shutdown");
await WaitForAsync(() => store.ObservedCancellation, timeoutMs: 2000);
store.ObservedCancellation.ShouldBeTrue("the in-flight save was cancelled through the engine's shutdown token");
}
[Fact]
public async Task Dispose_still_waits_for_a_save_that_completes_inside_the_bound()
{
// Positive control for the bound: a save that finishes promptly is drained, not cut off.
var up = new FakeUpstream();
up.Set("Temp", 50);
var store = new NeverReturningSaveStore { ReleaseAfter = TimeSpan.FromMilliseconds(100) };
var eng = Build(up, store);
eng.DisposeDrainTimeout = TimeSpan.FromSeconds(5);
await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
store.BlockNext = true;
up.Push("Temp", 150);
await WaitForAsync(() => store.SaveInProgress);
eng.Dispose();
store.SaveInProgress.ShouldBeFalse();
store.ObservedCancellation.ShouldBeFalse("a save that completes within the bound is never cancelled");
store.Saved.ShouldContain("A");
}
private static async Task WaitForAsync(Func<bool> cond, int timeoutMs = 5000)
{
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
while (DateTime.UtcNow < deadline)
{
if (cond()) return;
await Task.Delay(20);
}
throw new TimeoutException("Condition did not become true in time");
}
/// <summary>A store whose next SaveAsync blocks until cancelled (or until <see cref="ReleaseAfter"/>).</summary>
private sealed class NeverReturningSaveStore : IAlarmStateStore
{
private readonly InMemoryAlarmStateStore _inner = new();
public volatile bool BlockNext;
public volatile bool SaveInProgress;
public volatile bool ObservedCancellation;
public TimeSpan? ReleaseAfter { get; set; }
public List<string> Saved { get; } = [];
public Task<AlarmConditionState?> LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct);
public Task<IReadOnlyList<AlarmConditionState>> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct);
public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct);
public async Task SaveAsync(AlarmConditionState state, CancellationToken ct)
{
if (BlockNext)
{
BlockNext = false;
SaveInProgress = true;
try
{
await Task.Delay(ReleaseAfter ?? Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
ObservedCancellation = true;
throw;
}
finally
{
SaveInProgress = false;
}
}
lock (Saved) Saved.Add(state.AlarmId);
await _inner.SaveAsync(state, ct).ConfigureAwait(false);
}
}
}