6cbf763719
- R05:1/12 ScriptedAlarmEngine: _valueCache is cleared in LoadAsync (it is re-seeded for the referenced paths) so retired tags no longer accrete across deploys; OnUpstreamChange no longer writes the cache after Dispose. - R05:3 ScriptedAlarmEngine.Dispose drains in-flight work for at most 5 s (DisposeDrainTimeout), then cancels it through a shutdown token every DrainDirtyAsync/ShelvingCheckAsync now runs on, logs a Warning and gives one second of grace — a stalled IAlarmStateStore.SaveAsync can no longer hang host shutdown. A save that completes inside the bound is still drained (positive control test). - R05:6 ScriptedAlarmSource logs a throwing OnAlarmEvent subscriber at Warning (naming the alarm) instead of swallowing it. - R05:7 LocalDbStoreAndForwardSink keeps a cached dead-letter counter in step with every dead-letter/revive/purge path and resyncs it with the queued counter on the same cadence; GetStatus no longer runs a COUNT(*) scan. - R05:9 ScriptLogRateLimiter (20 per source per minute, fixed window, one summary entry naming the dropped count when the window rolls) behind ScriptLogTopicSink (key = script|vtag|alarm) and ScriptLogCompanionSink (key = script name). Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
160 lines
6.6 KiB
C#
160 lines
6.6 KiB
C#
using Serilog;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests;
|
|
|
|
/// <summary>
|
|
/// Review R05:1 / R05:12 / R05:3 — the engine's value cache is pruned on every reload (it used to keep
|
|
/// every path ever referenced, forever), is no longer written after Dispose, and Dispose's in-flight
|
|
/// drain is bounded (a hung store no longer hangs host shutdown).
|
|
/// </summary>
|
|
[Trait("Category", "Unit")]
|
|
public sealed class ScriptedAlarmEngineLifecycleTests
|
|
{
|
|
private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
|
|
new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
|
|
Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);
|
|
|
|
private static ScriptedAlarmEngine Build(FakeUpstream up, IAlarmStateStore? store = null)
|
|
{
|
|
var logger = new LoggerConfiguration().CreateLogger();
|
|
return new ScriptedAlarmEngine(up, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Reload_prunes_value_cache_entries_for_paths_the_new_alarm_set_no_longer_references()
|
|
{
|
|
var up = new FakeUpstream();
|
|
up.Set("Temp", 50);
|
|
up.Set("Pressure", 5);
|
|
using var eng = Build(up);
|
|
var ct = TestContext.Current.CancellationToken;
|
|
|
|
await eng.LoadAsync(
|
|
[Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100 || (int)ctx.GetTag("Pressure").Value > 10;""")], ct);
|
|
eng.ValueCacheCountForTest.ShouldBe(2);
|
|
|
|
// The redeploy drops Pressure from every predicate: its entry must go with it.
|
|
await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], ct);
|
|
eng.ValueCacheCountForTest.ShouldBe(1);
|
|
eng.ValueCacheContainsForTest("Pressure").ShouldBeFalse();
|
|
eng.ValueCacheContainsForTest("Temp").ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_push_after_Dispose_does_not_write_the_value_cache()
|
|
{
|
|
var up = new FakeUpstream();
|
|
up.Set("Temp", 50);
|
|
var eng = Build(up);
|
|
await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")],
|
|
TestContext.Current.CancellationToken);
|
|
eng.Dispose();
|
|
eng.ValueCacheCountForTest.ShouldBe(1);
|
|
|
|
eng.OnUpstreamChange("Late", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow));
|
|
|
|
eng.ValueCacheContainsForTest("Late").ShouldBeFalse("a disposed engine must not grow its cache");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Dispose_gives_up_on_a_store_that_never_returns_after_the_drain_bound_and_cancels_it()
|
|
{
|
|
var up = new FakeUpstream();
|
|
up.Set("Temp", 50);
|
|
var store = new NeverReturningSaveStore();
|
|
var eng = Build(up, store);
|
|
eng.DisposeDrainTimeout = TimeSpan.FromMilliseconds(300);
|
|
await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")],
|
|
TestContext.Current.CancellationToken);
|
|
|
|
store.BlockNext = true;
|
|
up.Push("Temp", 150); // Activated → SaveAsync → blocks forever (until cancelled)
|
|
await WaitForAsync(() => store.SaveInProgress);
|
|
|
|
var sw = System.Diagnostics.Stopwatch.StartNew();
|
|
eng.Dispose();
|
|
sw.Stop();
|
|
|
|
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the drain is bounded; a hung store cannot hang shutdown");
|
|
await WaitForAsync(() => store.ObservedCancellation, timeoutMs: 2000);
|
|
store.ObservedCancellation.ShouldBeTrue("the in-flight save was cancelled through the engine's shutdown token");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Dispose_still_waits_for_a_save_that_completes_inside_the_bound()
|
|
{
|
|
// Positive control for the bound: a save that finishes promptly is drained, not cut off.
|
|
var up = new FakeUpstream();
|
|
up.Set("Temp", 50);
|
|
var store = new NeverReturningSaveStore { ReleaseAfter = TimeSpan.FromMilliseconds(100) };
|
|
var eng = Build(up, store);
|
|
eng.DisposeDrainTimeout = TimeSpan.FromSeconds(5);
|
|
await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")],
|
|
TestContext.Current.CancellationToken);
|
|
|
|
store.BlockNext = true;
|
|
up.Push("Temp", 150);
|
|
await WaitForAsync(() => store.SaveInProgress);
|
|
|
|
eng.Dispose();
|
|
|
|
store.SaveInProgress.ShouldBeFalse();
|
|
store.ObservedCancellation.ShouldBeFalse("a save that completes within the bound is never cancelled");
|
|
store.Saved.ShouldContain("A");
|
|
}
|
|
|
|
private static async Task WaitForAsync(Func<bool> cond, int timeoutMs = 5000)
|
|
{
|
|
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
if (cond()) return;
|
|
await Task.Delay(20);
|
|
}
|
|
throw new TimeoutException("Condition did not become true in time");
|
|
}
|
|
|
|
/// <summary>A store whose next SaveAsync blocks until cancelled (or until <see cref="ReleaseAfter"/>).</summary>
|
|
private sealed class NeverReturningSaveStore : IAlarmStateStore
|
|
{
|
|
private readonly InMemoryAlarmStateStore _inner = new();
|
|
public volatile bool BlockNext;
|
|
public volatile bool SaveInProgress;
|
|
public volatile bool ObservedCancellation;
|
|
public TimeSpan? ReleaseAfter { get; set; }
|
|
public List<string> Saved { get; } = [];
|
|
|
|
public Task<AlarmConditionState?> LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct);
|
|
public Task<IReadOnlyList<AlarmConditionState>> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct);
|
|
public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct);
|
|
|
|
public async Task SaveAsync(AlarmConditionState state, CancellationToken ct)
|
|
{
|
|
if (BlockNext)
|
|
{
|
|
BlockNext = false;
|
|
SaveInProgress = true;
|
|
try
|
|
{
|
|
await Task.Delay(ReleaseAfter ?? Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
ObservedCancellation = true;
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
SaveInProgress = false;
|
|
}
|
|
}
|
|
lock (Saved) Saved.Add(state.AlarmId);
|
|
await _inner.SaveAsync(state, ct).ConfigureAwait(false);
|
|
}
|
|
}
|
|
}
|