Merge local master 0d49eaf7 into remed/lane-AbLegacy

Refs #738
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 15:06:25 -04:00
156 changed files with 16252 additions and 1242 deletions
@@ -0,0 +1,117 @@
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Configuration.Validation;
namespace ZB.MOM.WW.OtOpcUa.Configuration.Tests;
/// <summary>
/// R03:3.2 on the real provider: the migrated config database allows SNAPSHOT isolation (migration
/// <c>EnableSnapshotIsolation</c>) and <see cref="DraftSnapshotFactory.FromConfigDbAsync"/> reads under
/// it. The InMemory tests cannot see either — the provider has no transactions — so this is the one
/// place the deploy path's "one consistent read" is proven against SQL Server.
/// </summary>
[Trait("Category", "SchemaCompliance")]
[Collection(nameof(SchemaComplianceCollection))]
public sealed class DraftSnapshotFactorySqlServerTests
{
private readonly SchemaComplianceFixture _fixture;
/// <summary>Initializes a new instance of the <see cref="DraftSnapshotFactorySqlServerTests"/> class.</summary>
/// <param name="fixture">The migrated test database.</param>
public DraftSnapshotFactorySqlServerTests(SchemaComplianceFixture fixture) => _fixture = fixture;
/// <summary>The migration leaves the database with snapshot isolation allowed.</summary>
[Fact]
public void Migrated_database_allows_snapshot_isolation()
{
using var conn = _fixture.OpenConnection();
using var cmd = new SqlCommand(
"SELECT snapshot_isolation_state_desc FROM sys.databases WHERE name = DB_NAME();", conn);
cmd.ExecuteScalar().ShouldBe("ON");
}
/// <summary>
/// A row committed by another connection while the snapshot read is in progress is NOT observed by
/// the tables read after it — every table reflects the instant the transaction started. Positive
/// control: a read started after the commit does see the row.
/// </summary>
[Fact]
public async Task FromConfigDb_reads_every_table_as_of_one_instant()
{
var clusterId = $"SNAP-{Guid.NewGuid():N}"[..24];
var scriptId = $"late-{Guid.NewGuid():N}";
await using (var seed = NewContext())
{
seed.ServerClusters.Add(new ServerCluster
{
ClusterId = clusterId, Name = "snapshot-read", Enterprise = "zb", Site = "test",
NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
});
await seed.SaveChangesAsync();
}
try
{
var options = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
.UseSqlServer(_fixture.ConnectionString)
.AddInterceptors(new LateEditInterceptor(_fixture.ConnectionString, clusterId, scriptId))
.Options;
await using (var db = new OtOpcUaConfigDbContext(options))
{
var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(db);
// The interceptor committed a Script while ServerClusters (the FIRST table read) was being
// materialised. Scripts is read later in the same transaction and must not see it.
snapshot.ServerClusters.ShouldContain(c => c.ClusterId == clusterId);
snapshot.Scripts.ShouldNotContain(s => s.ScriptId == scriptId);
}
await using (var verify = NewContext())
{
// Positive control: the edit really landed, and a read started afterwards sees it.
(await verify.Scripts.AnyAsync(s => s.ScriptId == scriptId)).ShouldBeTrue();
var later = await DraftSnapshotFactory.FromConfigDbAsync(verify);
later.Scripts.ShouldContain(s => s.ScriptId == scriptId);
}
}
finally
{
await using var cleanup = NewContext();
await cleanup.Scripts.Where(s => s.ScriptId == scriptId).ExecuteDeleteAsync();
await cleanup.ServerClusters.Where(c => c.ClusterId == clusterId).ExecuteDeleteAsync();
}
}
private OtOpcUaConfigDbContext NewContext() => new(
new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseSqlServer(_fixture.ConnectionString).Options);
/// <summary>
/// Commits a Script on a SEPARATE connection the first time the snapshot read materialises the
/// seeded ServerCluster row — i.e. mid-read, after the first table and before Scripts.
/// </summary>
private sealed class LateEditInterceptor(string connectionString, string clusterId, string scriptId)
: IMaterializationInterceptor
{
private int _fired;
public object InitializedInstance(MaterializationInterceptionData materializationData, object entity)
{
if (entity is ServerCluster c && c.ClusterId == clusterId && Interlocked.Exchange(ref _fired, 1) == 0)
{
using var other = new OtOpcUaConfigDbContext(
new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseSqlServer(connectionString).Options);
other.Scripts.Add(new Script
{
ScriptId = scriptId, Name = "late edit", SourceCode = "return 1;", SourceHash = "late-hash",
});
other.SaveChanges();
}
return entity;
}
}
}
@@ -103,6 +103,36 @@ public sealed class DraftSnapshotFactoryTests : IDisposable
snapshot.ActiveReservations[0].ReleasedAt.ShouldBeNull();
}
/// <summary>
/// R03:3.2 — the deploy path validates AND composes from ONE snapshot, so the tables the composer
/// needs beyond the validator's set (clusters, nodes, API keys) must ride the same read.
/// </summary>
[Fact]
public async Task FromConfigDb_carries_clusters_nodes_and_api_keys_for_the_composer()
{
_db.ServerClusters.Add(new ServerCluster
{
ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
});
_db.ClusterNodes.Add(new ClusterNode
{
NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1",
ApplicationUri = "urn:OtOpcUa:site-a-1", Enabled = true, CreatedBy = "test",
});
_db.ApiKeys.Add(new ApiKey
{
ApiKeyId = "k1", Name = "scada", SecretHash = "hash", Principal = "scadabridge", Enabled = true,
});
await _db.SaveChangesAsync();
var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(_db);
snapshot.ServerClusters.ShouldHaveSingleItem().ClusterId.ShouldBe("SITE-A");
snapshot.ClusterNodes.ShouldHaveSingleItem().NodeId.ShouldBe("site-a-1:4053");
snapshot.ApiKeys.ShouldHaveSingleItem().ApiKeyId.ShouldBe("k1");
}
private void SeedEquipment(string equipmentId)
{
var uuid = Guid.NewGuid();
@@ -55,7 +55,10 @@ public sealed class DraftValidatorCalculationTests
Tags = [CalcTagRow("t-1", "dev-1", "a", "{\"changeTriggered\":true}")],
};
DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationScriptMissing" && e.Context == "t-1");
var errors = DraftValidator.Validate(draft);
errors.ShouldContain(e => e.Code == "CalculationScriptMissing" && e.Context == "t-1");
errors.ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger",
"an unresolved script has no reads to count — the missing script is the one error to report");
}
[Fact]
@@ -70,7 +73,10 @@ public sealed class DraftValidatorCalculationTests
Scripts = [],
};
DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationScriptNotFound" && e.Context == "t-1");
var errors = DraftValidator.Validate(draft);
errors.ShouldContain(e => e.Code == "CalculationScriptNotFound" && e.Context == "t-1");
errors.ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger",
"an unresolved script has no reads to count — the missing script is the one error to report");
}
[Fact]
@@ -82,7 +88,9 @@ public sealed class DraftValidatorCalculationTests
DriverInstances = [CalcDriver()],
Devices = [Device("dev-1", "di-calc", "engine")],
Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))],
Scripts = [ScriptRow("s1", "return 42.0;")],
// A read, so the change trigger is effective (a constant change-triggered script with no timer
// is now itself a deploy error — CalculationNoEffectiveTrigger).
Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value + 42.0;")],
};
DraftValidator.Validate(draft).ShouldNotContain(e => e.Code.StartsWith("Calculation"));
@@ -132,6 +140,58 @@ public sealed class DraftValidatorCalculationTests
msg.ShouldContain("calc/engine/b");
}
// ── effective-trigger gate (R07:U5) ─────────────────────────────────────────────────────
[Fact]
public void Change_triggered_tag_with_no_literal_read_and_no_timer_is_rejected()
{
var draft = new DraftSnapshot
{
GenerationId = 1, ClusterId = "c",
DriverInstances = [CalcDriver()],
Devices = [Device("dev-1", "di-calc", "engine")],
Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))],
Scripts = [ScriptRow("s1", "return ctx.Now.Hour;")],
};
DraftValidator.Validate(draft).ShouldContain(e =>
e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1" && e.Message.Contains("no literal ctx.GetTag"));
}
[Fact]
public void Tag_with_change_trigger_off_and_no_timer_is_rejected()
{
var draft = new DraftSnapshot
{
GenerationId = 1, ClusterId = "c",
DriverInstances = [CalcDriver()],
Devices = [Device("dev-1", "di-calc", "engine")],
Tags = [CalcTagRow("t-1", "dev-1", "a", "{\"scriptId\":\"s1\",\"changeTriggered\":false}")],
Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")],
};
DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1");
}
[Theory]
[InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true,\"timerIntervalMs\":1000}", "return ctx.Now.Hour;")] // timer rescues a read-less script
[InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":false,\"timerIntervalMs\":500}", "return 1.0;")] // timer-only tag
[InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // change trigger with a read
[InlineData("{\"scriptId\":\"s1\"}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // changeTriggered defaults to true
public void A_tag_with_an_effective_trigger_is_accepted(string cfg, string source)
{
var draft = new DraftSnapshot
{
GenerationId = 1, ClusterId = "c",
DriverInstances = [CalcDriver()],
Devices = [Device("dev-1", "di-calc", "engine")],
Tags = [CalcTagRow("t-1", "dev-1", "a", cfg)],
Scripts = [ScriptRow("s1", source)],
};
DraftValidator.Validate(draft).ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger");
}
[Fact]
public void Cross_driver_ref_is_terminal_not_an_edge()
{
@@ -342,6 +342,40 @@ public sealed class LocalDbStoreAndForwardSinkTests : IDisposable
sink.GetStatus().DeadLetterDepth.ShouldBe(0, "purged past retention");
}
/// <summary>
/// Review R05:7 — <c>GetStatus</c> no longer runs a <c>COUNT(*)</c> table scan per call for the
/// dead-letter depth: it reads a cached counter kept in step by every dead-letter / revive / purge
/// path (the other tests in this class pin those), and resynced from storage on the same cadence as
/// the queued-row counter. A row flipped underneath the sink (the peer replicating a dead-letter in)
/// is therefore invisible until the next resync — that is the documented trade, and the resync is
/// what makes it bounded rather than permanent.
/// </summary>
[Fact]
public async Task DeadLetterDepth_is_served_from_a_cached_counter_that_the_periodic_resync_corrects()
{
var writer = new FakeWriter();
using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log);
await sink.EnqueueAsync(Event("A1"), CancellationToken.None);
await sink.EnqueueAsync(Event("A2"), CancellationToken.None);
sink.GetStatus().DeadLetterDepth.ShouldBe(0);
var probesBefore = sink.DebugCapacityProbeCount;
// Drift the table underneath the sink, exactly as a replicated dead-letter from the pair peer would.
await Db.ExecuteAsync(
"UPDATE alarm_sf_events SET dead_lettered = 1 WHERE id IN (SELECT id FROM alarm_sf_events LIMIT 1)",
null, CancellationToken.None);
// Cached: GetStatus did not go to the table, so it still reports the pre-drift depth…
sink.GetStatus().DeadLetterDepth.ShouldBe(0, "GetStatus serves the cached counter, not a live COUNT(*)");
sink.DebugCapacityProbeCount.ShouldBe(probesBefore, "GetStatus must not probe storage");
// …and the resync (the one the enqueue cadence drives) brings both counters back in line.
await sink.DebugResyncCountersAsync(CancellationToken.None);
var status = sink.GetStatus();
status.DeadLetterDepth.ShouldBe(1);
status.QueueDepth.ShouldBe(1);
}
/// <summary>Verifies that RetryDeadLettered requeues dead-lettered rows for retry.</summary>
[Fact]
public async Task RetryDeadLettered_requeues_for_retry()
@@ -0,0 +1,162 @@
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests;
/// <summary>
/// Review R05:1 / R05:12 / R05:3 — the engine's value cache is pruned on every reload (it used to keep
/// every path ever referenced, forever), is no longer written after Dispose, and Dispose's in-flight
/// drain is bounded (a hung store no longer hangs host shutdown).
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptedAlarmEngineLifecycleTests
{
private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);
private static ScriptedAlarmEngine Build(FakeUpstream up, IAlarmStateStore? store = null)
{
var logger = new LoggerConfiguration().CreateLogger();
return new ScriptedAlarmEngine(up, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
}
[Fact]
public async Task Reload_prunes_value_cache_entries_for_paths_the_new_alarm_set_no_longer_references()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
up.Set("Pressure", 5);
using var eng = Build(up);
var ct = TestContext.Current.CancellationToken;
await eng.LoadAsync(
[Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100 || (int)ctx.GetTag("Pressure").Value > 10;""")], ct);
eng.ValueCacheCountForTest.ShouldBe(2);
// The redeploy drops Pressure from every predicate: its entry must go with it.
await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], ct);
eng.ValueCacheCountForTest.ShouldBe(1);
eng.ValueCacheContainsForTest("Pressure").ShouldBeFalse();
eng.ValueCacheContainsForTest("Temp").ShouldBeTrue();
}
[Fact]
public async Task A_push_after_Dispose_does_not_write_the_value_cache()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
var eng = Build(up);
await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
eng.Dispose();
eng.ValueCacheCountForTest.ShouldBe(1);
eng.OnUpstreamChange("Late", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow));
eng.ValueCacheContainsForTest("Late").ShouldBeFalse("a disposed engine must not grow its cache");
}
[Fact]
public async Task Dispose_gives_up_on_a_store_that_never_returns_after_the_drain_bound_and_cancels_it()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
var store = new NeverReturningSaveStore();
var eng = Build(up, store);
eng.DisposeDrainTimeout = TimeSpan.FromMilliseconds(300);
await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
store.BlockNext = true;
up.Push("Temp", 150); // Activated → SaveAsync → blocks forever (until cancelled)
await WaitForAsync(() => store.SaveInProgress);
var sw = System.Diagnostics.Stopwatch.StartNew();
eng.Dispose();
sw.Stop();
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the drain is bounded; a hung store cannot hang shutdown");
await WaitForAsync(() => store.ObservedCancellation, timeoutMs: 2000);
store.ObservedCancellation.ShouldBeTrue("the in-flight save was cancelled through the engine's shutdown token");
eng.ShutdownCtsDisposedForTest.ShouldBeTrue("the cancelled save completed inside the grace, so the CTS is released");
}
[Fact]
public async Task Dispose_still_waits_for_a_save_that_completes_inside_the_bound()
{
// Positive control for the bound: a save that finishes promptly is drained, not cut off.
var up = new FakeUpstream();
up.Set("Temp", 50);
var store = new NeverReturningSaveStore { ReleaseAfter = TimeSpan.FromMilliseconds(100) };
var eng = Build(up, store);
eng.DisposeDrainTimeout = TimeSpan.FromSeconds(5);
await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
store.BlockNext = true;
up.Push("Temp", 150);
await WaitForAsync(() => store.SaveInProgress);
eng.Dispose();
store.SaveInProgress.ShouldBeFalse();
store.ObservedCancellation.ShouldBeFalse("a save that completes within the bound is never cancelled");
store.Saved.ShouldContain("A");
eng.ShutdownCtsDisposedForTest.ShouldBeTrue("every background task completed, so the shutdown CTS is disposed");
Should.NotThrow(() => eng.OnUpstreamChange("Temp", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow)));
}
private static async Task WaitForAsync(Func<bool> cond, int timeoutMs = 5000)
{
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
while (DateTime.UtcNow < deadline)
{
if (cond()) return;
await Task.Delay(20);
}
throw new TimeoutException("Condition did not become true in time");
}
/// <summary>A store whose next SaveAsync blocks until cancelled (or until <see cref="ReleaseAfter"/>).</summary>
private sealed class NeverReturningSaveStore : IAlarmStateStore
{
private readonly InMemoryAlarmStateStore _inner = new();
public volatile bool BlockNext;
public volatile bool SaveInProgress;
public volatile bool ObservedCancellation;
public TimeSpan? ReleaseAfter { get; set; }
public List<string> Saved { get; } = [];
public Task<AlarmConditionState?> LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct);
public Task<IReadOnlyList<AlarmConditionState>> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct);
public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct);
public async Task SaveAsync(AlarmConditionState state, CancellationToken ct)
{
if (BlockNext)
{
BlockNext = false;
SaveInProgress = true;
try
{
await Task.Delay(ReleaseAfter ?? Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
ObservedCancellation = true;
throw;
}
finally
{
SaveInProgress = false;
}
}
lock (Saved) Saved.Add(state.AlarmId);
await _inner.SaveAsync(state, ct).ConfigureAwait(false);
}
}
}
@@ -0,0 +1,150 @@
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests;
/// <summary>
/// <see cref="ScriptedAlarmEngine"/> runs every predicate through a <see cref="ScriptRunGuard"/> keyed by
/// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation
/// spawned another abandoned run); now the orphan is one, later pushes are refused (and a
/// predicate that times out repeatedly is quarantined) — frozen at its prior state and annotated Bad quality so the operator can see it — while a
/// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine.
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptedAlarmEngineRunGuardTests
{
private const uint StatusBadInternalError = 0x80020000u;
private const string SpinPredicate =
"var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return false;";
private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);
private static ScriptedAlarmEngine Build(FakeUpstream up, ScriptingOptions? scripting = null)
{
var logger = new LoggerConfiguration().CreateLogger();
// 500 ms, not the 60 ms the guard's own tests use: the HEALTHY alarm's first run is JIT-cold and
// must finish inside the budget on a loaded CI box, or the positive control itself times out. The
// spin predicate outlives this by 12x, so the runaway alarm still cannot finish first.
return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
scriptTimeout: TimeSpan.FromMilliseconds(500), scripting: scripting);
}
[Fact]
public async Task Runaway_predicate_leaks_one_orphan_is_refused_annotated_Bad_once_and_a_sibling_keeps_evaluating()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
using var eng = Build(up);
var events = new List<ScriptedAlarmEvent>();
eng.OnEvent += (_, e) => { lock (events) events.Add(e); };
// The load itself evaluates once (startup recovery) — that is the first, abandoned run.
await eng.LoadAsync(
[Alarm("runaway", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"),
Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
// N more pushes while the orphan runs: every one is refused (a refusal is not a strike — I1), and
// the Bad annotation is emitted ONCE for the whole streak, not once per push (I3).
const int pushes = 4;
for (var i = 0; i < pushes; i++)
{
up.Push("Temp", 60 + i);
var expected = i + 1;
await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= expected);
}
await WaitForAsync(() => eng.RunGuard.StartedRunCount >= 1 + 1 + pushes);
eng.RunGuard.IsQuarantined("runaway", out _).ShouldBeFalse("refusals are not strikes");
eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one dispatch for the runaway alarm, one per evaluation for the healthy one");
eng.RunGuard.InFlightCount.ShouldBe(1);
// The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving —
// exactly once across the whole run of non-completions.
ScriptedAlarmEvent[] snapshot;
lock (events) snapshot = events.ToArray();
var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray();
quality.Length.ShouldBe(1);
quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError);
eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented");
// Positive control: the healthy sibling still transitions on the same engine.
up.Push("Temp", 150);
await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active);
}
/// <summary>I3 — a QUARANTINED predicate is re-attempted on every push and refused before dispatch; the
/// Bad annotation must still be emitted once, not once per push.</summary>
[Fact]
public async Task A_quarantined_predicate_emits_QualityChanged_once_across_many_pushes()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
var events = new List<ScriptedAlarmEvent>();
eng.OnEvent += (_, e) => { lock (events) events.Add(e); };
await eng.LoadAsync(
[Alarm("q", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"),
Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 1000;""")],
TestContext.Current.CancellationToken);
eng.RunGuard.IsQuarantined("q", out _).ShouldBeTrue();
// Each push re-evaluates BOTH alarms (same input). The healthy sibling's dispatch count is the
// positive control proving every push was actually processed — so the quarantined alarm was
// re-attempted N times, not coalesced into one.
const int pushes = 5;
for (var i = 0; i < pushes; i++)
{
var before = eng.RunGuard.StartedRunCount;
up.Push("Temp", 60 + i);
await WaitForAsync(() => eng.RunGuard.StartedRunCount > before);
}
eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one runaway dispatch, then healthy-only dispatches");
await Task.Delay(300); // absence window for trailing emissions
ScriptedAlarmEvent[] snapshot;
lock (events) snapshot = events.ToArray();
var quality = snapshot.Where(e => e.AlarmId == "q" && e.Emission == EmissionKind.QualityChanged).ToArray();
quality.Length.ShouldBe(1, "one Bad annotation for the quarantine, not one per push");
quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError);
snapshot.ShouldNotContain(e => e.AlarmId == "healthy" && e.Emission == EmissionKind.QualityChanged);
}
[Fact]
public async Task A_reload_lifts_the_quarantine()
{
var up = new FakeUpstream();
up.Set("Temp", 50);
using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
await eng.LoadAsync([Alarm("a", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}")],
TestContext.Current.CancellationToken);
eng.RunGuard.IsQuarantined("a", out _).ShouldBeTrue();
// The operator fixed the predicate and redeployed.
await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
eng.RunGuard.IsQuarantined("a", out _).ShouldBeFalse();
up.Push("Temp", 150);
await WaitForAsync(() => eng.GetState("a")!.Active == AlarmActiveState.Active);
}
private static async Task WaitForAsync(Func<bool> cond, int timeoutMs = 10_000)
{
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
while (DateTime.UtcNow < deadline)
{
if (cond()) return;
await Task.Delay(25);
}
throw new TimeoutException("Condition did not become true in time");
}
}
@@ -0,0 +1,59 @@
using Serilog;
using Serilog.Core;
using Serilog.Events;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests;
/// <summary>Review R05:6 — a throwing <see cref="ScriptedAlarmSource.OnAlarmEvent"/> subscriber used to be
/// swallowed with no diagnostic: a lost alarm event with no trace. It is now logged at Warning.</summary>
[Trait("Category", "Unit")]
public sealed class ScriptedAlarmSourceLoggingTests
{
private sealed class CapturingSink : ILogEventSink
{
public List<LogEvent> Events { get; } = [];
public void Emit(LogEvent logEvent) { lock (Events) Events.Add(logEvent); }
}
[Fact]
public async Task A_throwing_subscriber_is_logged_at_Warning_and_does_not_crash_the_adapter()
{
var sinkCapture = new CapturingSink();
var logger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sinkCapture).CreateLogger();
var up = new FakeUpstream();
up.Set("Temp", 50);
var engine = new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
await engine.LoadAsync(
[new ScriptedAlarmDefinition("Plant/Line1::HighTemp", "Plant/Line1", "HighTemp", AlarmKind.LimitAlarm,
AlarmSeverity.High, "Temp {Temp}C", """return (int)ctx.GetTag("Temp").Value > 100;""")],
TestContext.Current.CancellationToken);
using var _e = engine;
using var source = new ScriptedAlarmSource(engine, logger);
await source.SubscribeAlarmsAsync(Array.Empty<string>(), TestContext.Current.CancellationToken);
source.OnAlarmEvent += (_, _) => throw new InvalidOperationException("subscriber broke");
up.Push("Temp", 150);
var deadline = DateTime.UtcNow.AddSeconds(10);
while (DateTime.UtcNow < deadline)
{
lock (sinkCapture.Events)
{
if (sinkCapture.Events.Any(e => e.Level == LogEventLevel.Warning && e.Exception?.Message == "subscriber broke"))
break;
}
await Task.Delay(25);
}
LogEvent warning;
lock (sinkCapture.Events)
warning = sinkCapture.Events.Single(e => e.Level == LogEventLevel.Warning && e.Exception?.Message == "subscriber broke");
warning.RenderMessage().ShouldContain("Plant/Line1::HighTemp");
// The adapter survived: the engine's state still moved.
engine.GetState("Plant/Line1::HighTemp")!.Active.ShouldBe(AlarmActiveState.Active);
}
}
@@ -0,0 +1,141 @@
using Serilog;
using Serilog.Core;
using Serilog.Events;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests;
/// <summary>
/// Per-source rate limit on the script-log fan-out (review R05:9 / R02:P6): a script that logs — or
/// fails — on every evaluation of a 1 Hz input used to put one entry per second per script onto the
/// cluster bus and the main log. The limiter admits the first 20 entries per source per minute, drops
/// the rest, and surfaces one summary line (with the dropped count) when the next entry arrives after
/// the window rolled — so nothing is silently lost, and a chatty script cannot shed everyone else's
/// entries out of the bounded telemetry channels.
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptLogRateLimiterTests
{
private sealed class Clock
{
public DateTime Now = new(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
public void Advance(TimeSpan by) => Now += by;
}
[Fact]
public void Admits_the_first_N_per_window_then_suppresses_and_summarises_once_the_window_rolls()
{
var clock = new Clock();
var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => clock.Now);
limiter.Admit("s1").Admitted.ShouldBeTrue();
limiter.Admit("s1").Admitted.ShouldBeTrue();
limiter.Admit("s1").Admitted.ShouldBeTrue();
limiter.Admit("s1").Admitted.ShouldBeFalse();
limiter.Admit("s1").Admitted.ShouldBeFalse();
limiter.Admit("s1").SuppressedInPreviousWindow.ShouldBe(0, "no summary until the window rolls");
clock.Advance(TimeSpan.FromMinutes(1));
var first = limiter.Admit("s1");
first.Admitted.ShouldBeTrue("a fresh window admits again");
first.SuppressedInPreviousWindow.ShouldBe(3, "the caller emits one summary for the three dropped entries");
limiter.Admit("s1").SuppressedInPreviousWindow.ShouldBe(0, "the summary is reported exactly once");
}
[Fact]
public void Sources_are_independent()
{
var limiter = new ScriptLogRateLimiter(limitPerWindow: 1);
limiter.Admit("a").Admitted.ShouldBeTrue();
limiter.Admit("a").Admitted.ShouldBeFalse();
limiter.Admit("b").Admitted.ShouldBeTrue("a chatty source must not shed another source's entries");
}
[Fact]
public void Defaults_are_20_per_minute()
{
ScriptLogRateLimiter.DefaultLimitPerWindow.ShouldBe(20);
ScriptLogRateLimiter.DefaultWindow.ShouldBe(TimeSpan.FromMinutes(1));
var limiter = new ScriptLogRateLimiter();
limiter.LimitPerWindow.ShouldBe(20);
new ScriptLogRateLimiter(limitPerWindow: 3).LimitPerWindow.ShouldBe(3);
new ScriptLogRateLimiter(limitPerWindow: 0).LimitPerWindow.ShouldBe(1, "values below 1 are treated as 1");
for (var i = 0; i < 20; i++) limiter.Admit("s").Admitted.ShouldBeTrue();
limiter.Admit("s").Admitted.ShouldBeFalse();
}
// ── the two sinks ────────────────────────────────────────────────────────────────────────
private sealed class FakePublisher : IScriptLogPublisher
{
public List<ScriptLogEntry> Published { get; } = [];
public void Publish(ScriptLogEntry entry) => Published.Add(entry);
}
private static LogEvent Evt(LogEventLevel level, string template, params (string Key, string Val)[] props)
{
var tmpl = new Serilog.Parsing.MessageTemplateParser().Parse(template);
var logProps = props.Select(p => new LogEventProperty(p.Key, new ScalarValue(p.Val)));
return new LogEvent(DateTimeOffset.UtcNow, level, exception: null, tmpl, logProps);
}
[Fact]
public void Topic_sink_rate_limits_per_source_and_publishes_a_summary_entry()
{
var clock = new Clock();
var publisher = new FakePublisher();
var sink = new ScriptLogTopicSink(publisher, LogEventLevel.Information,
new ScriptLogRateLimiter(limitPerWindow: 2, clock: () => clock.Now));
for (var i = 0; i < 5; i++)
sink.Emit(Evt(LogEventLevel.Warning, "chatty", (ScriptLoggerFactory.ScriptIdProperty, "S1"), (ScriptLoggerFactory.VirtualTagIdProperty, "V1")));
sink.Emit(Evt(LogEventLevel.Warning, "other", (ScriptLoggerFactory.ScriptIdProperty, "S2")));
publisher.Published.Count(e => e.ScriptId == "S1").ShouldBe(2, "the limit is per source");
publisher.Published.Count(e => e.ScriptId == "S2").ShouldBe(1, "another source is unaffected");
clock.Advance(TimeSpan.FromMinutes(1));
sink.Emit(Evt(LogEventLevel.Warning, "chatty again", (ScriptLoggerFactory.ScriptIdProperty, "S1"), (ScriptLoggerFactory.VirtualTagIdProperty, "V1")));
var s1 = publisher.Published.Where(e => e.ScriptId == "S1").ToList();
s1.Count.ShouldBe(4, "2 admitted + 1 summary + 1 admitted in the new window");
var summary = s1[2];
summary.Message.ShouldContain("3");
summary.Message.ShouldContain("suppressed");
summary.Message.ShouldContain("limit 2/window", Case.Sensitive, "the summary names the CONFIGURED limit, not the default");
summary.VirtualTagId.ShouldBe("V1", "the summary carries the source's identity so the page attributes it");
summary.Level.ShouldBe("Warning");
s1[3].Message.ShouldBe("chatty again");
}
private sealed class CapturingSink : ILogEventSink
{
public List<LogEvent> Events { get; } = [];
public void Emit(LogEvent logEvent) => Events.Add(logEvent);
}
[Fact]
public void Companion_sink_rate_limits_per_script_and_mirrors_a_summary_line()
{
var clock = new Clock();
var main = new CapturingSink();
var mainLogger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(main).CreateLogger();
var sink = new ScriptLogCompanionSink(mainLogger, LogEventLevel.Error,
new ScriptLogRateLimiter(limitPerWindow: 2, clock: () => clock.Now));
for (var i = 0; i < 5; i++)
sink.Emit(Evt(LogEventLevel.Error, "boom", (ScriptLoggerFactory.ScriptNameProperty, "HighTemp")));
main.Events.Count.ShouldBe(2);
clock.Advance(TimeSpan.FromMinutes(1));
sink.Emit(Evt(LogEventLevel.Error, "boom", (ScriptLoggerFactory.ScriptNameProperty, "HighTemp")));
main.Events.Count.ShouldBe(4);
var summary = main.Events[2].RenderMessage();
summary.ShouldContain("HighTemp");
summary.ShouldContain("3");
summary.ShouldContain("suppressed");
}
}
@@ -0,0 +1,409 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests;
/// <summary>
/// <see cref="ScriptRunGuard"/> — the per-script run quarantine shared by the virtual-tag evaluator, the
/// Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so
/// before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard
/// refuses to start a run while the previous one is still executing, quarantines a script after three
/// consecutive timeouts (a refusal is not a strike), and quarantines a script whose single run allocates past the budget.
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptRunGuardTests
{
/// <summary>
/// A CPU-bound script that outlives every timeout in this file but still ENDS (so the orphaned pool
/// thread it leaves behind frees itself after the test). <c>System.DateTime</c> is outside the sandbox
/// deny-list.
/// </summary>
private const string SpinScript =
"""
var end = System.DateTime.UtcNow.AddSeconds(6);
while (System.DateTime.UtcNow < end) { }
return 1;
""";
private const string FastScript = "return 42;";
/// <summary>Outlives the 60 ms budget but finishes within ~250 ms, so the slot frees up between attempts.</summary>
private const string BriefSpinScript =
"""
var end = System.DateTime.UtcNow.AddMilliseconds(250);
while (System.DateTime.UtcNow < end) { }
return 1;
""";
private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60);
private static ScriptEvaluator<FakeScriptContext, int> Compile(string source) =>
ScriptEvaluator<FakeScriptContext, int>.Compile(source);
private static Task<int> Run(ScriptRunGuard guard, string key, string source,
ScriptEvaluator<FakeScriptContext, int> evaluator, TimeSpan? timeout = null, CancellationToken ct = default) =>
guard.RunAsync(key, source, evaluator, new FakeScriptContext(), timeout ?? TimeSpan.FromSeconds(5), ct);
[Fact]
public async Task Completed_run_returns_the_value_and_releases_the_in_flight_slot()
{
var guard = new ScriptRunGuard();
using var fast = Compile(FastScript);
var value = await Run(guard, "vt-1", FastScript, fast, ct: TestContext.Current.CancellationToken);
value.ShouldBe(42);
guard.InFlightCount.ShouldBe(0);
guard.StartedRunCount.ShouldBe(1);
guard.IsQuarantined("vt-1", out _).ShouldBeFalse();
}
[Fact]
public async Task A_timed_out_run_throws_ScriptTimeoutException_and_keeps_the_slot_held_by_the_orphan()
{
var guard = new ScriptRunGuard();
using var spin = Compile(SpinScript);
await Should.ThrowAsync<ScriptTimeoutException>(() =>
Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));
// The abandoned script is still executing on its pool thread — the guard knows, the caller does not.
guard.InFlightCount.ShouldBe(1);
guard.StartedRunCount.ShouldBe(1);
}
[Fact]
public async Task A_run_while_the_previous_orphan_is_still_executing_is_refused_without_starting_another()
{
var guard = new ScriptRunGuard();
using var spin = Compile(SpinScript);
await Should.ThrowAsync<ScriptTimeoutException>(() =>
Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));
var ex = await Should.ThrowAsync<ScriptRunRefusedException>(() =>
Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));
ex.Message.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
guard.StartedRunCount.ShouldBe(1, "the refusal must not dispatch a second Task.Run");
guard.InFlightCount.ShouldBe(1);
guard.RefusedWhileRunningCount.ShouldBe(1);
}
[Fact]
public async Task Three_consecutive_timeouts_quarantine_the_script_and_a_sibling_keeps_evaluating()
{
var guard = new ScriptRunGuard();
using var spin = Compile(BriefSpinScript);
using var fast = Compile(FastScript);
var ct = TestContext.Current.CancellationToken;
// Three timeouts in a row — each orphan is allowed to finish so the next attempt is a real run,
// not a refusal (a refusal is not a strike; see the next test).
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await WaitForIdle(guard);
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await WaitForIdle(guard);
var quarantined = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
await WaitForIdle(guard);
quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
quarantined.Message.ShouldContain("3 consecutive");
guard.IsQuarantined("bad", out var reason).ShouldBeTrue();
reason.ShouldNotBeNull();
guard.QuarantinedCount.ShouldBe(1);
// A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run.
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct));
guard.StartedRunCount.ShouldBe(3);
// Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key.
(await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42);
guard.IsQuarantined("good", out _).ShouldBeFalse();
guard.StartedRunCount.ShouldBe(4);
}
/// <summary>
/// A refusal is NOT a strike (review fix round 1, I1): on a 1 Hz trigger with a 2 s budget a single
/// 4 s overrun is one timeout followed by two refusals, and counting those would quarantine a script
/// that merely ran long once. A refusal is a Bad publish plus its counter, and it leaves the timeout
/// streak untouched — neither adding to it nor resetting it.
/// </summary>
[Fact]
public async Task Refusals_neither_count_toward_nor_reset_the_timeout_streak()
{
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 });
using var spin = Compile(BriefSpinScript);
var ct = TestContext.Current.CancellationToken;
// timeout #1, then refusals while the orphan is still running — however many, no quarantine.
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
for (var i = 0; i < 5; i++)
await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
guard.IsQuarantined("k", out _).ShouldBeFalse("refusals are not strikes");
guard.RefusedWhileRunningCount.ShouldBe(5);
await WaitForIdle(guard);
// …and they did not RESET the streak either: the second real timeout reaches the threshold of 2.
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct));
guard.IsQuarantined("k", out _).ShouldBeTrue();
await WaitForIdle(guard);
}
/// <summary>
/// The thread-leak regression, and the stuck-orphan quarantine (fix round 2). Pre-guard, a
/// <c>while(true)</c> script on a 1 Hz trigger leaked one pool thread per trigger — 30 orphans after 30
/// triggers, trending toward pool starvation. Now the first trigger's orphan is the only one; later
/// triggers are refused while it is young, and once it has run longer than
/// <see cref="ScriptingOptions.StuckOrphanQuarantineFactor"/> × the timeout (10 × 60 ms here — the
/// trigger cadence is scaled down from 1 Hz to keep the test short) the next refusal quarantines it.
/// A healthy sibling keeps evaluating throughout (the positive control an absence assertion needs).
/// </summary>
/// <remarks>
/// The load-bearing assertions are on the guard's own bookkeeping (<see cref="ScriptRunGuard.InFlightCount"/>
/// / <see cref="ScriptRunGuard.StartedRunCount"/>): <see cref="ThreadPool.ThreadCount"/> moves with
/// every parallel test class on a loaded machine, so it is kept as a coarse secondary check only —
/// it must stay far below the 30 the leak would have produced.
/// </remarks>
[Fact]
public async Task Runaway_script_leaves_one_orphan_is_quarantined_once_stuck_and_a_sibling_keeps_evaluating()
{
var guard = new ScriptRunGuard();
using var spin = Compile(SpinScript);
using var fast = Compile(FastScript);
var ct = TestContext.Current.CancellationToken;
var threadsBefore = ThreadPool.ThreadCount;
var stuckAfter = ShortTimeout * ScriptingOptions.DefaultStuckOrphanQuarantineFactor; // 600 ms
var healthyResults = new List<int>();
var outcomes = new List<string>();
var sw = System.Diagnostics.Stopwatch.StartNew();
for (var trigger = 0; trigger < 30; trigger++)
{
try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); outcomes.Add("value"); }
catch (ScriptTimeoutException) { outcomes.Add("timeout"); }
catch (ScriptRunRefusedException) { outcomes.Add(sw.Elapsed < stuckAfter ? "refused-young" : "refused"); }
catch (ScriptQuarantinedException) { outcomes.Add("quarantined"); }
healthyResults.Add(await Run(guard, "healthy", FastScript, fast, ct: ct));
await Task.Delay(50, ct);
}
guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing");
guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling");
outcomes[0].ShouldBe("timeout");
outcomes.ShouldContain("refused-young", "a young orphan is only refused — a short overrun is not quarantined");
guard.IsQuarantined("runaway", out var reason).ShouldBeTrue("the orphan outlived 10x its budget: it is stuck");
reason!.ShouldContain("stuck");
outcomes[^1].ShouldBe("quarantined");
guard.QuarantinedCount.ShouldBe(1);
healthyResults.ShouldAllBe(v => v == 42);
healthyResults.Count.ShouldBe(30);
var threadDelta = ThreadPool.ThreadCount - threadsBefore;
threadDelta.ShouldBeLessThan(15, "thirty leaked orphans would have grown the pool by ~30 threads");
}
/// <summary>
/// Positive control for the stuck rule: a 2x overrun (one timeout, refusals while the orphan is young)
/// is NOT quarantined, and once the orphan completes the script evaluates normally again.
/// </summary>
[Fact]
public async Task A_2x_overrun_is_not_quarantined_and_resumes_evaluating_once_the_orphan_completes()
{
var guard = new ScriptRunGuard();
const string source =
"""
var ms = (int)(ctx.GetTag("spinMs").Value ?? 0);
var end = System.DateTime.UtcNow.AddMilliseconds(ms);
while (System.DateTime.UtcNow < end) { }
return 7;
""";
using var eval = Compile(source);
var ct = TestContext.Current.CancellationToken;
var timeout = TimeSpan.FromMilliseconds(100);
FakeScriptContext Ctx(int spinMs)
{
var c = new FakeScriptContext();
c.Tags["spinMs"] = new Core.Abstractions.DataValueSnapshot(spinMs, 0u, c.Now, c.Now);
return c;
}
// Warm the JIT so the first measured run's overrun is the script's, not the compiler's.
(await guard.RunAsync("k", source, eval, Ctx(0), TimeSpan.FromSeconds(5), ct)).ShouldBe(7);
// 2x overrun: 200 ms against a 100 ms budget — times out, then is refused while it finishes.
await Should.ThrowAsync<ScriptTimeoutException>(() => guard.RunAsync("k", source, eval, Ctx(200), timeout, ct));
await Should.ThrowAsync<ScriptRunRefusedException>(() => guard.RunAsync("k", source, eval, Ctx(0), timeout, ct));
await WaitForIdle(guard);
guard.IsQuarantined("k", out _).ShouldBeFalse("a 2x overrun is far below the 10x stuck threshold");
(await guard.RunAsync("k", source, eval, Ctx(0), timeout, ct)).ShouldBe(7, "the script resumes evaluating");
guard.QuarantinedCount.ShouldBe(0);
}
/// <summary>The stuck rule is a knob: with <see cref="ScriptingOptions.StuckOrphanQuarantineFactor"/> ≤ 0 a
/// stuck orphan is only ever refused.</summary>
[Fact]
public async Task StuckOrphanQuarantineFactor_zero_disables_the_stuck_rule()
{
var guard = new ScriptRunGuard(new ScriptingOptions { StuckOrphanQuarantineFactor = 0 });
using var spin = Compile(SpinScript);
var ct = TestContext.Current.CancellationToken;
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct));
await Task.Delay(ShortTimeout * 12, ct);
await Should.ThrowAsync<ScriptRunRefusedException>(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct));
guard.IsQuarantined("k", out _).ShouldBeFalse();
}
[Fact]
public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot()
{
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
using var spin = Compile(SpinScript);
using var fast = Compile(FastScript);
var ct = TestContext.Current.CancellationToken;
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct));
guard.IsQuarantined("vt", out _).ShouldBeTrue();
// The operator fixed the script: a different source identity under the same key starts clean. The
// old orphan is a sunk cost — it keeps running, but it must not block the FIXED script.
(await Run(guard, "vt", FastScript, fast, ct: ct)).ShouldBe(42);
guard.IsQuarantined("vt", out _).ShouldBeFalse();
guard.InFlightCount.ShouldBe(1, "the abandoned orphan is still counted until it finishes");
}
[Fact]
public async Task Reset_and_ResetAll_clear_a_quarantine()
{
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
using var spin = Compile(SpinScript);
using var fast = Compile(FastScript);
var ct = TestContext.Current.CancellationToken;
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "a", SpinScript, spin, ShortTimeout, ct));
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "b", SpinScript, spin, ShortTimeout, ct));
guard.QuarantinedKeys.ShouldBe(new[] { "a", "b" }, ignoreOrder: true);
guard.Reset("a");
guard.IsQuarantined("a", out _).ShouldBeFalse();
guard.IsQuarantined("b", out _).ShouldBeTrue();
guard.ResetAll();
guard.IsQuarantined("b", out _).ShouldBeFalse();
// The key is usable again (a fresh slot: the orphan from the quarantined run no longer blocks it).
(await Run(guard, "b", FastScript, fast, ct: ct)).ShouldBe(42);
}
[Fact]
public async Task A_run_that_allocates_past_the_budget_is_quarantined()
{
var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 1024 * 1024 });
// Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the
// allocation as dead code (which is exactly what it does to the naive one-liner).
const string hungry =
"long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;";
using var evaluator = Compile(hungry);
using var fast = Compile(FastScript);
var ct = TestContext.Current.CancellationToken;
var ex = await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "hog", hungry, evaluator, ct: ct));
ex.Message.ShouldContain("allocat");
guard.IsQuarantined("hog", out var reason).ShouldBeTrue();
reason.ShouldNotBeNull();
reason.ShouldContain("allocat");
// Never dispatched again, and the slot was released (the run DID complete).
await Should.ThrowAsync<ScriptQuarantinedException>(() => Run(guard, "hog", hungry, evaluator, ct: ct));
guard.StartedRunCount.ShouldBe(1);
guard.InFlightCount.ShouldBe(0);
// Positive control: a modest allocation under the budget is fine.
(await Run(guard, "ok", FastScript, fast, ct: ct)).ShouldBe(42);
}
[Fact]
public async Task A_disabled_allocation_budget_never_quarantines()
{
var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 0 });
// Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the
// allocation as dead code (which is exactly what it does to the naive one-liner).
const string hungry =
"long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;";
using var evaluator = Compile(hungry);
(await Run(guard, "hog", hungry, evaluator, ct: TestContext.Current.CancellationToken)).ShouldBe(32 * (256 * 1024 + 1));
guard.IsQuarantined("hog", out _).ShouldBeFalse();
}
[Fact]
public async Task A_script_that_throws_propagates_unwrapped_and_counts_as_a_completion()
{
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 });
const string throwing = "throw new System.InvalidOperationException(\"boom\");";
using var thrower = Compile(throwing);
// A spin that outlives the timeout but finishes quickly, so the slot frees up between attempts.
const string briefSpin =
"""
var end = System.DateTime.UtcNow.AddMilliseconds(250);
while (System.DateTime.UtcNow < end) { }
return 1;
""";
using var spin = Compile(briefSpin);
var ct = TestContext.Current.CancellationToken;
// timeout #1
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct));
await WaitForIdle(guard);
// A throw is a completion — the script returned control — so the consecutive counter resets…
var ex = await Should.ThrowAsync<InvalidOperationException>(() => Run(guard, "k", throwing, thrower, ct: ct));
ex.Message.ShouldBe("boom");
guard.InFlightCount.ShouldBe(0);
// …which is why this second timeout does NOT reach the threshold of 2.
await Should.ThrowAsync<ScriptTimeoutException>(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct));
guard.IsQuarantined("k", out _).ShouldBeFalse();
await WaitForIdle(guard);
}
[Fact]
public async Task Caller_cancellation_wins_over_the_timeout()
{
var guard = new ScriptRunGuard();
using var spin = Compile(SpinScript);
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(80));
await Should.ThrowAsync<OperationCanceledException>(() =>
Run(guard, "k", SpinScript, spin, TimeSpan.FromSeconds(5), cts.Token));
guard.IsQuarantined("k", out _).ShouldBeFalse();
}
[Fact]
public async Task Quarantine_is_observable_through_the_event_and_the_counter()
{
var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
using var spin = Compile(SpinScript);
var seen = new List<(string Key, string Reason)>();
guard.Quarantined += (_, e) => seen.Add((e.Key, e.Reason));
await Should.ThrowAsync<ScriptQuarantinedException>(() =>
Run(guard, "k", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken));
seen.ShouldHaveSingleItem();
seen[0].Key.ShouldBe("k");
seen[0].Reason.ShouldContain("consecutive");
guard.QuarantinedCount.ShouldBe(1);
}
private static async Task WaitForIdle(ScriptRunGuard guard)
{
var deadline = DateTime.UtcNow.AddSeconds(5);
while (guard.InFlightCount != 0 && DateTime.UtcNow < deadline)
await Task.Delay(10);
guard.InFlightCount.ShouldBe(0, "the brief spin must have finished");
}
}
@@ -0,0 +1,148 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests;
/// <summary>
/// Sandbox hardening (review R05:10): <c>System.Threading</c> is denied as a namespace with a
/// type-granular allow-list (<c>CancellationToken</c>, <c>Interlocked</c>, <c>Volatile</c>) so a
/// script can no longer park its pool thread on a blocking primitive or touch a system-global kernel
/// object; <c>System.Type</c>'s reflection entry points (<c>GetType(string)</c>, <c>GetMethod*</c>,
/// <c>InvokeMember</c>, …) and <c>System.Runtime.CompilerServices.RuntimeHelpers</c> are denied so a
/// forbidden type cannot be reached by name without ever being named syntactically.
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptSandboxHardeningTests
{
private static ScriptSandboxViolationException Rejects(string source) =>
Should.Throw<ScriptSandboxViolationException>(() =>
ScriptEvaluator<FakeScriptContext, int>.Compile(source));
// ── System.Threading blocking / synchronisation primitives ───────────────────────────────
[Fact]
public void Rejects_SemaphoreSlim_Wait()
{
var ex = Rejects("new System.Threading.SemaphoreSlim(0).Wait(); return 0;");
ex.Rejections.ShouldContain(r => r.TypeName.Contains("SemaphoreSlim") && r.Namespace == "System.Threading");
}
[Fact]
public void Rejects_SpinWait_SpinUntil()
=> Rejects("System.Threading.SpinWait.SpinUntil(() => false); return 0;");
[Fact]
public void Rejects_Monitor()
=> Rejects("var o = new object(); System.Threading.Monitor.Enter(o); return 0;");
[Fact]
public void Rejects_named_Mutex_system_global_kernel_object()
=> Rejects("using var m = new System.Threading.Mutex(false, \"Global\\\\otopcua\"); return 0;");
[Fact]
public void Rejects_ManualResetEventSlim()
=> Rejects("var e = new System.Threading.ManualResetEventSlim(false); e.Wait(); return 0;");
[Fact]
public void Rejects_EventWaitHandle()
=> Rejects("var e = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset); return 0;");
[Fact]
public void Rejects_Thread_ThreadPool_and_Timer_still()
{
Rejects("var t = new System.Threading.Thread(() => { }); return 0;");
Rejects("System.Threading.ThreadPool.QueueUserWorkItem(_ => { }); return 0;");
Rejects("var t = new System.Threading.Timer(_ => { }); return 0;");
}
[Fact]
public void Rejects_Thread_Sleep()
=> Rejects("System.Threading.Thread.Sleep(10); return 0;");
// ── the allow-list inside System.Threading ───────────────────────────────────────────────
[Fact]
public async Task Allows_CancellationToken()
{
using var ev = ScriptEvaluator<FakeScriptContext, int>.Compile(
"var t = System.Threading.CancellationToken.None; return t.IsCancellationRequested ? 1 : 0;");
(await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(0);
}
[Fact]
public async Task Allows_Interlocked()
{
using var ev = ScriptEvaluator<FakeScriptContext, int>.Compile(
"var x = 41; System.Threading.Interlocked.Increment(ref x); return x;");
(await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(42);
}
[Fact]
public async Task Allows_Volatile()
{
using var ev = ScriptEvaluator<FakeScriptContext, int>.Compile(
"var x = 42; return System.Threading.Volatile.Read(ref x);");
(await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(42);
}
// ── System.Type reflection entry points ──────────────────────────────────────────────────
[Fact]
public void Rejects_Type_GetType_by_name()
{
var ex = Rejects("var t = System.Type.GetType(\"System.IO.File\"); return t is null ? 0 : 1;");
ex.Rejections.ShouldContain(r => r.Message.Contains("GetType"));
}
[Fact]
public void Rejects_Type_GetMethod()
=> Rejects("var m = typeof(string).GetMethod(\"Trim\"); return m is null ? 0 : 1;");
[Fact]
public void Rejects_Type_GetMethods()
=> Rejects("return typeof(string).GetMethods().Length;");
[Fact]
public void Rejects_Type_InvokeMember()
=> Rejects("""return (int)typeof(string).InvokeMember("Length", System.Reflection.BindingFlags.GetProperty, null, "abc", null)!;""");
[Fact]
public void Rejects_Type_GetProperty_GetField_GetMember_GetConstructor_GetEvent()
{
Rejects("return typeof(string).GetProperty(\"Length\") is null ? 0 : 1;");
Rejects("return typeof(string).GetField(\"Empty\") is null ? 0 : 1;");
Rejects("return typeof(string).GetMember(\"Length\").Length;");
Rejects("return typeof(string).GetConstructors().Length;");
Rejects("return typeof(string).GetEvent(\"x\") is null ? 0 : 1;");
}
[Fact]
public async Task Allows_instance_GetType_and_Type_metadata_reads()
{
// object.GetType() is declared on System.Object, and Type.Name / IsValueType are plain metadata
// reads — neither is a reflection entry point, and both are common in authored scripts.
using var ev = ScriptEvaluator<FakeScriptContext, int>.Compile(
"object o = 42; var t = o.GetType(); return t.IsValueType && typeof(int).Name == \"Int32\" ? 1 : 0;");
(await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(1);
}
// ── RuntimeHelpers ───────────────────────────────────────────────────────────────────────
[Fact]
public void Rejects_RuntimeHelpers()
{
var ex = Rejects(
"var o = System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(string)); return o is null ? 0 : 1;");
ex.Rejections.ShouldContain(r => r.TypeName.Contains("RuntimeHelpers"));
}
[Fact]
public async Task Allows_compile_time_only_CompilerServices_attributes()
{
// The namespace stays open for the harmless compile-time types (MethodImplOptions etc.).
using var ev = ScriptEvaluator<FakeScriptContext, int>.Compile(
"var o = System.Runtime.CompilerServices.MethodImplOptions.NoInlining; return (int)o;");
(await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(8);
}
}
@@ -33,6 +33,27 @@ public sealed class FocasCommandBaseValidationTests
/// <summary>Invokes option validation without interval.</summary>
public void InvokeValidateNoInterval() => ValidateOptions(intervalMs: null);
/// <summary>Invokes the address/data-type check.</summary>
public void InvokeValidateDataType(string address, FocasDataType type) => ValidateDataType(address, type);
}
/// <summary>
/// #740 / R07:U8 — <c>--type String</c> on a PMC / PARAM / MACRO address is refused at the CLI boundary
/// (the driver would skip the tag and the read would print BadNodeIdUnknown); a fixed-tree path keeps it.
/// </summary>
[Theory]
[InlineData("R100", FocasDataType.String, false)]
[InlineData("MACRO:500", FocasDataType.String, false)]
[InlineData("PARAM:6711", FocasDataType.String, false)]
[InlineData("R100", FocasDataType.Byte, true)]
[InlineData("FIXED:Identity/SeriesNumber", FocasDataType.String, true)]
public void Validate_data_type_against_address(string address, FocasDataType type, bool ok)
{
var sut = new Probe { CncHost = "host", IntervalMs = 1000 };
if (ok) Should.NotThrow(() => sut.InvokeValidateDataType(address, type));
else Should.Throw<CliFx.Exceptions.CommandException>(() => sut.InvokeValidateDataType(address, type))
.Message.ShouldContain("String");
}
/// <summary>Verifies that default options are accepted.</summary>
@@ -184,4 +184,24 @@ public sealed class WriteCommandParseValueTests
{
ReadCommand.SynthesiseTagName(symbol, type).ShouldBe(expected);
}
/// <summary>An array value splits on the separator and parses each element per --type (#739 U9 gate tooling).</summary>
[Fact]
public void ParseArrayValue_parses_each_element()
{
WriteCommand.ParseArrayValue("1, -2,3", TwinCATDataType.Int, 3, ",")
.ShouldBe(new object[] { (short)1, (short)-2, (short)3 });
WriteCommand.ParseArrayValue("a b|c", TwinCATDataType.String, 2, "|")
.ShouldBe(new object[] { "a b", "c" });
}
/// <summary>A count that differs from --array-length is refused before the PLC is touched.</summary>
[Fact]
public void ParseArrayValue_refuses_a_count_mismatch()
{
Should.Throw<CliFx.Exceptions.CommandException>(
() => WriteCommand.ParseArrayValue("1,2", TwinCATDataType.Int, 3, ","));
Should.Throw<CliFx.Exceptions.CommandException>(
() => WriteCommand.ParseArrayValue("1", TwinCATDataType.Int, 0, ","));
}
}
@@ -0,0 +1,189 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Serilog;
using Serilog.Events;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests;
/// <summary>
/// Review 07 Calculation remediation: a compile failure is negative-cached by source hash and its
/// Warning/script-log fires on the Good→Bad transition only (P4); inputs are recorded before
/// <c>InitializeAsync</c> so a Restart computes from current inputs (H6); array results dedup
/// structurally (H8); <c>ShutdownAsync</c> drops the compiled scripts (L2).
/// </summary>
public sealed class CalculationDriverRemediationTests
{
private const uint StatusGood = 0u;
private const uint StatusBadInternalError = 0x80020000u;
private static RawTagEntry CalcTag(string rawPath, string source, bool changeTriggered = true, int? timerMs = null)
{
var timer = timerMs is { } ms ? $",\"timerIntervalMs\":{ms}" : "";
var json = $"{{\"scriptId\":\"s-{rawPath}\",\"changeTriggered\":{(changeTriggered ? "true" : "false")}{timer}," +
$"\"scriptSource\":{System.Text.Json.JsonSerializer.Serialize(source)}}}";
return new RawTagEntry(rawPath, json, WriteIdempotent: false);
}
private static (CalculationDriver Driver, ConcurrentQueue<DataChangeEventArgs> Published, List<ScriptLogEntry> Logs, CapturingLogger HostLog)
Build(params RawTagEntry[] tags)
{
var logs = new List<ScriptLogEntry>();
var sink = new ScriptLogTopicSink(new CapturingPublisher(logs), LogEventLevel.Information);
var scriptRoot = new ScriptRootLogger(new LoggerConfiguration().WriteTo.Sink(sink).CreateLogger());
var hostLog = new CapturingLogger();
var driver = new CalculationDriver(new CalculationDriverOptions { RawTags = tags }, "calc-drv", scriptRoot, hostLog);
var published = new ConcurrentQueue<DataChangeEventArgs>();
driver.OnDataChange += (_, e) => published.Enqueue(e);
return (driver, published, logs, hostLog);
}
private static async Task Settle(CalculationDriver driver)
{
var deadline = DateTime.UtcNow.AddSeconds(30);
while (!driver.IsEvaluationIdle && DateTime.UtcNow < deadline) await Task.Delay(10);
driver.IsEvaluationIdle.ShouldBeTrue("the calc driver's evaluation pump should have drained");
}
/// <summary>Fix round 1 minor: Dispose releases the evaluation gate and its cancellation source — even
/// with a runaway run in flight (the guard lets the caller's cancel win, so the awaited run returns
/// promptly) — and a trigger arriving after Dispose is harmless.</summary>
[Fact]
public async Task Dispose_with_a_run_in_flight_disposes_the_eval_gate_and_a_late_trigger_is_harmless()
{
var (driver, _, _, _) = Build(
CalcTag("calc/e/spin", "var v = ctx.GetTag(\"src/a\").Value; var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 1;"));
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow);
var deadline = DateTime.UtcNow.AddSeconds(10);
while (driver.IsEvaluationIdle && DateTime.UtcNow < deadline) await Task.Delay(5);
driver.IsEvaluationIdle.ShouldBeFalse("positive control: a run is in flight when Dispose lands");
var sw = System.Diagnostics.Stopwatch.StartNew();
driver.Dispose();
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the caller's cancel releases the awaited run; Dispose is not held for the 6 s spin");
driver.EvalResourcesDisposedForTest.ShouldBeTrue();
Should.NotThrow(() => driver.OnDependencyValue("src/a", 2, StatusGood, DateTime.UtcNow));
Should.NotThrow(() => driver.Dispose());
}
[Fact]
public async Task A_compile_failure_is_remembered_and_logged_once_across_repeated_triggers()
{
var (driver, published, logs, hostLog) = Build(
CalcTag("calc/e/broken", "var x = (double)ctx.GetTag(\"src/a\").Value; this is not valid C#;"));
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
for (var i = 0; i < 5; i++)
{
driver.OnDependencyValue("src/a", i, StatusGood, DateTime.UtcNow);
await Settle(driver);
}
published.Count.ShouldBe(1, "Bad is published once per transition");
published.TryDequeue(out var bad).ShouldBeTrue();
bad!.Snapshot.StatusCode.ShouldBe(StatusBadInternalError);
logs.Count.ShouldBe(1, "the script-log entry fires on the Good→Bad transition only");
hostLog.Entries.Count(e => e.Level == LogLevel.Warning && e.Message.Contains("compile failed")).ShouldBe(1,
"Roslyn is consulted once per source; later triggers hit the negative cache");
driver.CompileFailureCountForTest.ShouldBe(1);
}
[Fact]
public async Task A_changed_failure_reason_is_logged_even_while_already_Bad()
{
// Throws with a different message depending on the input, so the failure REASON changes while the
// published status stays Bad — the operator must see the new reason.
var (driver, published, logs, _) = Build(
CalcTag("calc/e/thrower", "throw new System.InvalidOperationException(\"boom \" + (int)ctx.GetTag(\"src/a\").Value);"));
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow);
await Settle(driver);
driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); // same reason → no new log
await Settle(driver);
driver.OnDependencyValue("src/a", 2, StatusGood, DateTime.UtcNow); // new reason → logged
await Settle(driver);
published.Count.ShouldBe(1);
logs.Count.ShouldBe(2);
logs[0].Message.ShouldContain("boom 1");
logs[1].Message.ShouldContain("boom 2");
}
[Fact]
public async Task Inputs_delivered_before_InitializeAsync_are_kept_so_the_first_trigger_computes()
{
// H6: a Restart is a new driver whose adapter registers interest at construction; the mux's
// retained-value replay therefore lands BEFORE InitializeAsync. It used to be dropped.
var (driver, published, _, _) = Build(
CalcTag("calc/e/sum", "return (double)ctx.GetTag(\"src/a\").Value + (double)ctx.GetTag(\"src/b\").Value;"));
driver.OnDependencyValue("src/a", 10.0, StatusGood, DateTime.UtcNow); // replayed, pre-init
driver.OnDependencyValue("src/b", 20.0, StatusGood, DateTime.UtcNow);
published.ShouldBeEmpty("evaluation is gated on running");
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
driver.OnDependencyValue("src/a", 12.0, StatusGood, DateTime.UtcNow); // only ONE input moves after init
await Settle(driver);
published.Count.ShouldBe(1, "src/b's pre-init value satisfied the change-gate");
published.TryDequeue(out var e).ShouldBeTrue();
e!.Snapshot.Value.ShouldBe(32.0);
}
[Fact]
public async Task An_equal_array_result_is_deduped_structurally()
{
var (driver, published, _, _) = Build(
CalcTag("calc/e/arr", "var v = (double)ctx.GetTag(\"src/a\").Value; return new double[] { v, v * 2 };"));
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow);
await Settle(driver);
driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); // same array contents
await Settle(driver);
published.Count.ShouldBe(1, "a structurally equal array is not a change");
driver.OnDependencyValue("src/a", 2.0, StatusGood, DateTime.UtcNow);
await Settle(driver);
published.Count.ShouldBe(2, "a different array still publishes");
published.Last().Snapshot.Value.ShouldBe(new double[] { 2.0, 4.0 });
}
[Fact]
public async Task ShutdownAsync_drops_the_compiled_scripts()
{
var (driver, published, _, _) = Build(CalcTag("calc/e/x", "return (double)ctx.GetTag(\"src/a\").Value * 2.0;"));
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow);
await Settle(driver);
published.Count.ShouldBe(1);
driver.CompiledScriptCountForTest.ShouldBe(1);
await driver.ShutdownAsync(TestContext.Current.CancellationToken);
driver.CompiledScriptCountForTest.ShouldBe(0, "the collectible ALCs must not survive the generation");
}
private sealed class CapturingPublisher(List<ScriptLogEntry> entries) : IScriptLogPublisher
{
public void Publish(ScriptLogEntry entry) { lock (entries) entries.Add(entry); }
}
internal sealed class CapturingLogger : ILogger<CalculationDriver>
{
public ConcurrentBag<(LogLevel Level, string Message)> Entries { get; } = new();
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
Func<TState, Exception?, string> formatter)
=> Entries.Add((logLevel, formatter(state, exception)));
}
}
@@ -229,11 +229,13 @@ public sealed class CalculationDriverTests
logs[0].ScriptId.ShouldBe("s-calc/e/div");
logs[0].VirtualTagId.ShouldBe("calc/e/div");
// a = 0 again → still failing, but NOT a fresh transition → no second Bad publish (script-log still emits).
// a = 0 again → still failing for the SAME reason, NOT a fresh transition → no second Bad publish,
// and (R07:P4) no second script-log entry either: the log fires on the transition or on a change of
// reason, not on every trigger of a tag that is already Bad.
driver.OnDependencyValue("src/a", 0, StatusGood, DateTime.UtcNow);
await Settle(driver);
published.ShouldBeEmpty();
logs.Count.ShouldBe(2);
logs.Count.ShouldBe(1);
// a = 5 → 20 → Good recovery is force-published.
driver.OnDependencyValue("src/a", 5, StatusGood, DateTime.UtcNow);
@@ -0,0 +1,87 @@
using Microsoft.Extensions.Logging.Abstractions;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests;
/// <summary>
/// <see cref="CalculationEvaluator"/> runs every compiled script through a <see cref="ScriptRunGuard"/>
/// keyed by calc-tag RawPath. A timed-out run is abandoned on its pool thread; before the guard, every
/// later trigger of the same tag stacked another orphan behind it (one leaked thread per trigger). Now
/// every later trigger is refused ("previous evaluation still running" — Bad, but not a quarantine
/// strike; only consecutive timeouts quarantine), and a healthy sibling on the same evaluator keeps computing.
/// </summary>
public sealed class CalculationEvaluatorRunGuardTests
{
private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger());
/// <summary>Outlives the 60 ms budget by a wide margin but still ends, so the orphan frees itself.</summary>
private const string SpinScript =
"var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;";
private static readonly Dictionary<string, object?> NoDeps = new();
[Fact]
public async Task Triggers_during_the_previous_orphan_are_refused_not_quarantined_and_a_sibling_keeps_computing()
{
using var sut = new CalculationEvaluator(
NullLogger<CalculationEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60));
var ct = TestContext.Current.CancellationToken;
var first = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct);
first.Success.ShouldBeFalse();
first.Reason.ShouldNotBeNull();
first.Reason.ShouldContain("timed out");
// Every later trigger while the orphan runs is a Bad "previous evaluation still running" — a
// refusal is not a strike, so the tag is NOT quarantined by them (fix round 1, I1).
for (var i = 0; i < 3; i++)
{
var refused = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct);
refused.Success.ShouldBeFalse();
refused.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
}
sut.QuarantinedScripts.ShouldBeEmpty();
sut.RunGuard.RefusedWhileRunningCount.ShouldBe(3);
// Exactly one run was ever dispatched for the runaway tag — the leak is one orphan, not one per trigger.
sut.RunGuard.StartedRunCount.ShouldBe(1);
sut.RunGuard.InFlightCount.ShouldBe(1);
// Positive control: a sibling tag on the same evaluator still computes.
var sibling = await sut.EvaluateAsync("calc/ok", "return 42;", Snap(NoDeps), ct);
sibling.Success.ShouldBeTrue(sibling.Reason);
sibling.Value.ShouldBe(42);
}
[Fact]
public async Task ClearCompiledScripts_lifts_the_quarantine_as_a_redeploy_does()
{
using var sut = new CalculationEvaluator(
NullLogger<CalculationEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60),
new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
var ct = TestContext.Current.CancellationToken;
var r = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct);
r.Reason.ShouldNotBeNull();
r.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
sut.ClearCompiledScripts();
sut.QuarantinedScripts.ShouldBeEmpty();
// The fixed script (new source) runs; the old orphan is a sunk cost that no longer blocks the tag.
var fixedRun = await sut.EvaluateAsync("calc/loop", "return 7;", Snap(NoDeps), ct);
fixedRun.Success.ShouldBeTrue(fixedRun.Reason);
fixedRun.Value.ShouldBe(7);
}
private static Dictionary<string, Core.Abstractions.DataValueSnapshot> Snap(Dictionary<string, object?> deps)
{
var now = DateTime.UtcNow;
var d = new Dictionary<string, Core.Abstractions.DataValueSnapshot>(StringComparer.Ordinal);
foreach (var kv in deps) d[kv.Key] = new Core.Abstractions.DataValueSnapshot(kv.Value, 0u, now, now);
return d;
}
}
@@ -72,6 +72,155 @@ public sealed class WireBackendCoverageTests
}
}
/// <summary>
/// #740 / R07:H14 — a macro / parameter number above 32767 reaches the CNC unnarrowed. The mock decodes
/// the request's 32-bit <c>Arg1</c> as the number, so a decoy is seeded at the address the old
/// <c>(short)</c> cast produced (99999 → -31073, 40000 → -25536, as the unsigned key the mock sees):
/// reading the decoy's value means the number was wrapped. #500 / #6711 are the positive control that
/// ordinary numbers still read.
/// </summary>
[Fact]
public async Task Macro_and_parameter_numbers_above_short_range_read_the_authored_variable()
{
if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason);
var ct = TestContext.Current.CancellationToken;
static string Wrapped(int number) => ((uint)(int)(short)number).ToString(System.Globalization.CultureInfo.InvariantCulture);
await _fx.LoadProfileAsync("FWLIB64", ct);
await _fx.PatchStateAsync(new
{
parameters = new Dictionary<string, object>
{
["6711"] = new { type = "long", value = 1234, @decimal = 0 },
["40000"] = new { type = "long", value = 4242, @decimal = 0 },
[Wrapped(40000)] = new { type = "long", value = -1, @decimal = 0 },
},
macros = new Dictionary<string, object>
{
["500"] = new { value = 42000, @decimal = 3 },
["99999"] = new { value = 7, @decimal = 0 },
[Wrapped(99999)] = new { value = 666, @decimal = 0 },
},
}, ct);
var drv = new FocasDriver(new FocasDriverOptions
{
// ThirtyOne_i: the capability matrix admits #99999 (its documented ceiling); the parameter is read on
// an Unknown-series device, which admits any number.
Devices = [new FocasDeviceOptions(DeviceHost, Series: FocasCncSeries.ThirtyOne_i)],
RawTags = FocasRawTags.Entries(
new FocasTagDefinition("Macro500", DeviceHost, "MACRO:500", FocasDataType.Float64, Writable: false),
new FocasTagDefinition("Macro99999", DeviceHost, "MACRO:99999", FocasDataType.Int32, Writable: false),
new FocasTagDefinition("Param6711", DeviceHost, "PARAM:6711", FocasDataType.Int32, Writable: false)),
Probe = new FocasProbeOptions { Enabled = false },
}, driverInstanceId: "wire-wide-numbers", clientFactory: new WireFocasClientFactory());
var unknownSeries = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(DeviceHost)],
RawTags = FocasRawTags.Entries(
new FocasTagDefinition("Param40000", DeviceHost, "PARAM:40000", FocasDataType.Int32, Writable: false)),
Probe = new FocasProbeOptions { Enabled = false },
}, driverInstanceId: "wire-wide-param", clientFactory: new WireFocasClientFactory());
await using (drv)
await using (unknownSeries)
{
await drv.InitializeAsync("{}", ct);
var snaps = await drv.ReadAsync(["Macro500", "Macro99999", "Param6711"], ct);
snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.Good);
Convert.ToDouble(snaps[0].Value).ShouldBe(42.0, tolerance: 0.001);
snaps[1].Value.ShouldBe(7, "MACRO:99999 must read #99999, not the wrapped #-31073 decoy (666)");
Convert.ToInt32(snaps[2].Value).ShouldBe(1234);
await unknownSeries.InitializeAsync("{}", ct);
var param = await unknownSeries.ReadAsync(["Param40000"], ct);
param[0].StatusCode.ShouldBe(FocasStatusMapper.Good);
Convert.ToInt32(param[0].Value).ShouldBe(4242, "PARAM:40000 must not read the wrapped decoy (-1)");
}
}
/// <summary>
/// #740 / R07:H13 — re-initialising a live driver (the fixed-tree loop running, reads in flight over the real
/// wire) swaps the device table instead of clearing it under its readers: no reader throws, reads keep
/// returning Good, and shutdown leaves no session behind (a fresh driver can still connect afterwards).
/// </summary>
[Fact]
public async Task Reinitialise_under_live_reads_swaps_the_device_table()
{
if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason);
var ct = TestContext.Current.CancellationToken;
await _fx.LoadProfileAsync("FWLIB64", ct);
await _fx.PatchStateAsync(new
{
macros = new Dictionary<string, object> { ["500"] = new { value = 42000, @decimal = 3 } },
}, ct);
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(DeviceHost)],
RawTags = FocasRawTags.Entries(
new FocasTagDefinition("Macro500", DeviceHost, "MACRO:500", FocasDataType.Float64, Writable: false)),
Probe = new FocasProbeOptions { Enabled = false },
FixedTree = new FocasFixedTreeOptions
{
Enabled = true,
PollInterval = TimeSpan.FromMilliseconds(50),
ProgramPollInterval = TimeSpan.Zero,
TimerPollInterval = TimeSpan.Zero,
},
}, driverInstanceId: "wire-reinit-swap", clientFactory: new WireFocasClientFactory());
await using (drv)
{
await drv.InitializeAsync("{}", ct);
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
var errors = new ConcurrentQueue<Exception>();
var good = 0;
var reader = Task.Run(async () =>
{
while (!stop.IsCancellationRequested)
{
try
{
_ = drv.GetHostStatuses();
var s = await drv.ReadAsync(["Macro500", $"{DeviceHost}/Identity/SeriesNumber"], CancellationToken.None);
if (s[0].StatusCode == FocasStatusMapper.Good) Interlocked.Increment(ref good);
}
catch (Exception ex) { errors.Enqueue(ex); }
}
}, CancellationToken.None);
for (var i = 0; i < 5; i++)
{
await Task.Delay(150, ct);
await drv.InitializeAsync("{}", ct);
}
await Task.Delay(300, ct);
stop.Cancel();
await reader;
errors.ShouldBeEmpty();
good.ShouldBeGreaterThan(0, "positive control: reads went over the wire between re-initialises");
}
// A fresh driver still connects and reads Good after the re-initialise churn and the shutdown.
var after = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(DeviceHost)],
RawTags = FocasRawTags.Entries(
new FocasTagDefinition("Macro500", DeviceHost, "MACRO:500", FocasDataType.Float64, Writable: false)),
Probe = new FocasProbeOptions { Enabled = false },
}, driverInstanceId: "wire-reinit-after", clientFactory: new WireFocasClientFactory());
await using (after)
{
await after.InitializeAsync("{}", ct);
(await after.ReadAsync(["Macro500"], ct))[0].StatusCode.ShouldBe(FocasStatusMapper.Good);
}
}
/// <summary>Verifies that discover emits device folder and tag variables.</summary>
[Fact]
public async Task Discover_emits_device_folder_and_tag_variables()
@@ -0,0 +1,218 @@
using System.Net;
using System.Net.Sockets;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests.Series;
/// <summary>
/// #740 fix round 1 (review I2) — wire-level cases for H15 (fixed-tree health) and H20 (alarm record layout)
/// against the focas-mock fixture.
/// </summary>
[Collection(FocasSimCollection.Name)]
public sealed class WireBackendHealthAndLayoutTests
{
private readonly FocasSimFixture _fx;
/// <summary>Initializes a new instance of the test class with the FOCAS simulation fixture.</summary>
/// <param name="fx">The FOCAS simulation fixture.</param>
public WireBackendHealthAndLayoutTests(FocasSimFixture fx) => _fx = fx;
/// <summary>
/// A local TCP relay in front of the shared mock, so ONE test can take "the CNC" away and bring it back
/// without stopping a fixture other suites use: <see cref="Down"/> closes every relayed session and stops
/// listening (new connects are refused); <see cref="Up"/> listens again on the same port.
/// </summary>
private sealed class CncRelay : IAsyncDisposable
{
private readonly string _targetHost;
private readonly int _targetPort;
private readonly List<TcpClient> _live = [];
private TcpListener? _listener;
private CancellationTokenSource? _cts;
public CncRelay(string targetHost, int targetPort)
{
_targetHost = targetHost;
_targetPort = targetPort;
Port = Listen(0);
}
public int Port { get; private set; }
private int Listen(int port)
{
_cts = new CancellationTokenSource();
_listener = new TcpListener(IPAddress.Loopback, port);
_listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_listener.Start();
var ct = _cts.Token;
var listener = _listener;
_ = Task.Run(() => AcceptLoopAsync(listener, ct), CancellationToken.None);
return ((IPEndPoint)_listener.LocalEndpoint).Port;
}
private async Task AcceptLoopAsync(TcpListener listener, CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
TcpClient inbound;
try { inbound = await listener.AcceptTcpClientAsync(ct); }
catch { return; }
var outbound = new TcpClient();
try { await outbound.ConnectAsync(_targetHost, _targetPort, ct); }
catch { inbound.Dispose(); outbound.Dispose(); continue; }
lock (_live) { _live.Add(inbound); _live.Add(outbound); }
_ = Pump(inbound, outbound, ct);
_ = Pump(outbound, inbound, ct);
}
}
private static async Task Pump(TcpClient from, TcpClient to, CancellationToken ct)
{
try { await from.GetStream().CopyToAsync(to.GetStream(), ct); }
catch { /* the relay was taken down, or a side closed */ }
finally { from.Dispose(); to.Dispose(); }
}
public void Down()
{
_cts?.Cancel();
_listener?.Stop();
lock (_live)
{
foreach (var c in _live) c.Dispose();
_live.Clear();
}
}
/// <summary>
/// Listens again on the SAME port (the driver's host string names it). The re-bind is retried briefly: the
/// OS can hold the port for a moment after <see cref="Down"/>, or hand it to another process — the latter
/// surfaces as a clear failure after the retries rather than a random flake.
/// </summary>
public void Up()
{
for (var attempt = 1; ; attempt++)
{
try { Listen(Port); return; }
catch (SocketException) when (attempt < 20) { Thread.Sleep(100); }
}
}
public ValueTask DisposeAsync()
{
Down();
return ValueTask.CompletedTask;
}
}
/// <summary>
/// H15 over the wire: a driver serving only <c>FIXED:</c> tags is Healthy with a moving LastSuccessfulRead
/// while its CNC answers; when the CNC disappears it degrades with LastSuccessfulRead frozen (the #527
/// watchdog's conjunction); and it recovers to Healthy when the CNC returns (positive control).
/// </summary>
[Fact]
public async Task Fixed_only_driver_degrades_on_cnc_loss_and_recovers_over_the_wire()
{
if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason);
var ct = TestContext.Current.CancellationToken;
await _fx.LoadProfileAsync("FWLIB64", ct);
await using var relay = new CncRelay(_fx.Host, _fx.Port);
var host = $"focas://127.0.0.1:{relay.Port}";
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(host)],
RawTags = FocasRawTags.Entries(
new FocasTagDefinition("Series", host, "FIXED:Identity/SeriesNumber", FocasDataType.String, Writable: false)),
Probe = new FocasProbeOptions { Enabled = false },
Timeout = TimeSpan.FromSeconds(1),
FixedTree = new FocasFixedTreeOptions
{
PollInterval = TimeSpan.FromMilliseconds(50),
ProgramPollInterval = TimeSpan.Zero,
TimerPollInterval = TimeSpan.Zero,
},
}, driverInstanceId: "wire-fixed-health", clientFactory: new WireFocasClientFactory())
{
FixedTreeStaleBudgetOverride = TimeSpan.FromMilliseconds(500),
};
await using (drv)
{
await drv.InitializeAsync("{}", ct);
await WaitFor(() => drv.GetDeviceState(host)?.LastFixedPollUtc is not null, TimeSpan.FromSeconds(10));
var first = drv.GetHealth().LastSuccessfulRead!.Value;
await WaitFor(() => drv.GetHealth().LastSuccessfulRead > first, TimeSpan.FromSeconds(5));
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
relay.Down();
await WaitFor(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(15));
var frozen = drv.GetHealth().LastSuccessfulRead;
await Task.Delay(TimeSpan.FromSeconds(1), ct);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "nothing answered, so LastSuccessfulRead stays frozen");
(await drv.ReadAsync(["Series"], ct))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError,
"the stale fixed-tree value reads Bad on the same budget that degrades the driver");
// Positive control: the CNC returns ⇒ Healthy again, LastSuccessfulRead moves. The connect backoff
// (1 s doubling) bounds how soon the loop re-dials.
relay.Up();
await WaitFor(() => drv.GetHealth().State == DriverState.Healthy, TimeSpan.FromSeconds(40));
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(frozen!.Value);
(await drv.ReadAsync(["Series"], ct))[0].StatusCode.ShouldBe(FocasStatusMapper.Good);
}
}
/// <summary>
/// H20 over the wire: the mock answers <c>cnc_rdalmmsg2</c> in the 80-byte LEGACY layout. Nineteen active
/// alarms make a 1,520-byte payload — also exactly 20 × 76 — which the old length test decoded as twenty
/// garbage vendor records. Positive control: the unambiguous 3-alarm reply decodes the same alarms.
/// </summary>
[Fact]
public async Task Nineteen_legacy_alarms_decode_as_nineteen_over_the_wire()
{
if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason);
var ct = TestContext.Current.CancellationToken;
await _fx.LoadProfileAsync("FWLIB64", ct);
var client = new WireFocasClientFactory().Create();
try
{
await client.ConnectAsync(FocasHostAddress.TryParse(_fx.DeviceUri)!, TimeSpan.FromSeconds(2), ct);
await _fx.PatchStateAsync(new { alarms = Alarms(3) }, ct);
var three = await client.ReadAlarmsAsync(ct);
three.Count.ShouldBe(3);
three[2].AlarmNumber.ShouldBe(402);
await _fx.PatchStateAsync(new { alarms = Alarms(19) }, ct);
var nineteen = await client.ReadAlarmsAsync(ct);
nineteen.Count.ShouldBe(19, "a 1,520-byte legacy reply is 19 records, not 20 vendor ones");
nineteen.Select(a => a.AlarmNumber).ShouldBe(Enumerable.Range(400, 19));
nineteen[18].Message.ShouldBe("SERVO ALARM 418");
nineteen[18].Axis.ShouldBe((short)(1 + (18 % 4)));
}
finally
{
client.Dispose();
await _fx.PatchStateAsync(new { alarms = Array.Empty<object>() }, ct);
}
static object[] Alarms(int n) => [.. Enumerable.Range(0, n)
.Select(i => (object)new { alm_no = 400 + i, type = 2 + (i % 3), axis = 1 + (i % 4), msg = $"SERVO ALARM {400 + i}" })];
}
private static async Task WaitFor(Func<bool> condition, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
await Task.Delay(25);
}
}
}
@@ -259,7 +259,7 @@ internal sealed class FakeFocasClientFactory : IFocasClientFactory
public IFocasClient Create()
{
var c = Customise?.Invoke() ?? new FakeFocasClient();
Clients.Add(c);
lock (Clients) Clients.Add(c);
return c;
}
}
@@ -0,0 +1,114 @@
using System.Buffers.Binary;
using System.Text;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// #740 / R07:H20 — <c>ParseAlarms</c> picked the 76-byte vendor layout whenever <c>Length % 76 == 0</c>, so a
/// legacy 80-byte reply of exactly 19 records (1,520 B = 20 × 76) decoded as 20 garbage vendor records. The
/// response block carries no record count, so the layout is now chosen from the two things that are known: the
/// count the REQUEST asked for (a reply never holds more) and each layout's own field invariants.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasAlarmLayoutTests
{
private static byte[] Legacy(params (int no, int type, int axis, string msg)[] records)
{
var buf = new byte[records.Length * 80];
for (var i = 0; i < records.Length; i++)
{
var o = i * 80;
var (no, type, axis, msg) = records[i];
BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o, 4), no);
BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o + 4, 4), type);
BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o + 8, 4), axis);
BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o + 12, 4), msg.Length);
Encoding.ASCII.GetBytes(msg).CopyTo(buf.AsSpan(o + 16));
}
return buf;
}
private static byte[] Vendor(params (int no, short type, short axis, string msg)[] records)
{
var buf = new byte[records.Length * 76];
for (var i = 0; i < records.Length; i++)
{
var o = i * 76;
var (no, type, axis, msg) = records[i];
BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o, 4), no);
BinaryPrimitives.WriteInt16BigEndian(buf.AsSpan(o + 4, 2), type);
BinaryPrimitives.WriteInt16BigEndian(buf.AsSpan(o + 6, 2), axis);
BinaryPrimitives.WriteInt16BigEndian(buf.AsSpan(o + 10, 2), (short)msg.Length);
Encoding.ASCII.GetBytes(msg).CopyTo(buf.AsSpan(o + 12));
}
return buf;
}
private static (int, int, int, string)[] LegacySet(int n) =>
[.. Enumerable.Range(0, n).Select(i => (400 + i, 2 + (i % 3), 1 + (i % 4), $"SERVO ALARM {400 + i}"))];
private static (int, short, short, string)[] VendorSet(int n) =>
[.. Enumerable.Range(0, n).Select(i => (1000 + i, (short)(4 + (i % 3)), (short)(i % 2), $"OVERTRAVEL +{i}"))];
/// <summary>The ambiguous length: 19 legacy records decode as 19 legacy records, not 20 vendor ones.</summary>
[Fact]
public void Nineteen_legacy_records_decode_as_legacy()
{
var payload = Legacy(LegacySet(19));
payload.Length.ShouldBe(1520);
(payload.Length % 76).ShouldBe(0, "the length really is ambiguous");
var alarms = FocasWireClient.ParseAlarms(payload, 32);
alarms.Count.ShouldBe(19);
alarms[0].AlarmNumber.ShouldBe(400);
alarms[0].Message.ShouldBe("SERVO ALARM 400");
alarms[18].AlarmNumber.ShouldBe(418);
alarms[18].Axis.ShouldBe((short)(1 + (18 % 4)));
}
/// <summary>Positive control at the same length: 20 vendor records still decode as vendor.</summary>
[Fact]
public void Twenty_vendor_records_decode_as_vendor()
{
var payload = Vendor(VendorSet(20));
payload.Length.ShouldBe(1520);
var alarms = FocasWireClient.ParseAlarms(payload, 32);
alarms.Count.ShouldBe(20);
alarms[0].AlarmNumber.ShouldBe(1000);
alarms[19].Message.ShouldBe("OVERTRAVEL +19");
}
/// <summary>
/// Review I2 edge: vendor records with type 0 and EMPTY messages also satisfy the legacy invariants (the
/// int32 read of <c>type&lt;&lt;16 | axis</c> is just the axis, and the "length" int32 is the empty message's
/// zero bytes) — both readings are plausible, so the vendor default must hold and the numbers decode.
/// </summary>
[Fact]
public void Twenty_vendor_records_with_type_zero_and_empty_messages_stay_vendor()
{
var payload = Vendor([.. Enumerable.Range(0, 20).Select(i => (2000 + i, (short)0, (short)(i % 3), ""))]);
payload.Length.ShouldBe(1520);
var alarms = FocasWireClient.ParseAlarms(payload, 32);
alarms.Count.ShouldBe(20);
alarms.Select(a => a.AlarmNumber).ShouldBe(Enumerable.Range(2000, 20));
alarms.ShouldAllBe(a => a.Type == 0 && a.Message == "");
}
/// <summary>The unambiguous lengths are unchanged.</summary>
[Fact]
public void Unambiguous_lengths_are_unchanged()
{
FocasWireClient.ParseAlarms(Vendor(VendorSet(3)), 32).Count.ShouldBe(3);
FocasWireClient.ParseAlarms(Legacy(LegacySet(3)), 32).Count.ShouldBe(3);
FocasWireClient.ParseAlarms([], 32).ShouldBeEmpty();
Should.Throw<FocasWireException>(() => FocasWireClient.ParseAlarms(new byte[77], 32));
}
}
@@ -645,7 +645,8 @@ public sealed class FocasAlarmProjectionTests
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
// Value path Degraded, alarm path good ⇒ Degraded: an alarm success cannot mask it.
factory.Clients.Single().ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
// A COMMS-class status: a CNC refusal (e.g. BadOutOfRange) is not a value-path Degraded since #740.
factory.Clients.Single().ReadStatuses["R100"] = FocasStatusMapper.BadCommunicationError;
await drv.ReadAsync([valuePath], CancellationToken.None);
drv.NoteAlarmReadSucceeded(Host);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
@@ -0,0 +1,284 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// #740 / R07:P9 — <see cref="FocasDriver.ReadAsync"/> walked the whole batch sequentially, so a SLOW CNC
/// (TCP-accept-then-stall, the live 31i-B case) cost its timeout per tag per poll during which no other
/// device was read. Devices are now read concurrently, each still serialised on its own socket.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasConcurrentDeviceReadTests
{
private const string HostA = "focas://10.0.0.5:8193";
private const string HostB = "focas://10.0.0.6:8193";
/// <summary>
/// A client that records which device it dialled. Device A's read only completes once device B's read has
/// STARTED — so a sequential walk (A first) can never finish A, while a concurrent one does. Also counts
/// the peak number of reads in flight on any one client.
/// </summary>
private sealed class OrderingClient(TaskCompletionSource bStarted, int[] global) : FakeFocasClient
{
private string _host = "";
private int _inFlight;
public int PeakInFlight;
public int Reads;
public override Task ConnectAsync(FocasHostAddress address, TimeSpan timeout, CancellationToken ct)
{
_host = address.ToString();
return base.ConnectAsync(address, timeout, ct);
}
public override async Task<(object? value, uint status)> ReadAsync(
FocasAddress address, FocasDataType type, CancellationToken ct)
{
var now = Interlocked.Increment(ref _inFlight);
InterlockedMax(ref PeakInFlight, now);
Interlocked.Increment(ref Reads);
// global[0] = reads in flight across EVERY client, global[1] = its peak.
InterlockedMax(ref global[1], Interlocked.Increment(ref global[0]));
try
{
if (_host.Contains("10.0.0.6", StringComparison.Ordinal)) bStarted.TrySetResult();
else await bStarted.Task.WaitAsync(TimeSpan.FromSeconds(2), ct).ConfigureAwait(false);
await Task.Delay(20, ct).ConfigureAwait(false);
return ((sbyte)1, FocasStatusMapper.Good);
}
finally
{
Interlocked.Decrement(ref _inFlight);
Interlocked.Decrement(ref global[0]);
}
}
private static void InterlockedMax(ref int target, int value)
{
int seen;
while ((seen = Volatile.Read(ref target)) < value
&& Interlocked.CompareExchange(ref target, value, seen) != seen) { }
}
}
/// <summary>A stalled device no longer holds every other device's read hostage — and each device stays serialised.</summary>
[Fact]
public async Task Devices_are_read_concurrently_and_each_stays_serialised()
{
var bStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var global = new int[2];
var factory = new FakeFocasClientFactory { Customise = () => new OrderingClient(bStarted, global) };
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(HostA, DeviceName: "A"), new FocasDeviceOptions(HostB, DeviceName: "B")],
RawTags =
[
FocasRawTags.Entry(new FocasTagDefinition("raw/A1", "A", "R100", FocasDataType.Byte)),
FocasRawTags.Entry(new FocasTagDefinition("raw/A2", "A", "R101", FocasDataType.Byte)),
FocasRawTags.Entry(new FocasTagDefinition("raw/B1", "B", "R100", FocasDataType.Byte)),
],
Probe = new FocasProbeOptions { Enabled = false },
// Off so every tag takes the per-tag path — this pins device-level concurrency, not coalescing.
PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = false },
}, "focas-p9", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
var snaps = await drv.ReadAsync(["raw/A1", "raw/A2", "raw/B1"], CancellationToken.None);
snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.Good,
"device A's read can only finish once device B's has started — a sequential walk times A out");
factory.Clients.Count.ShouldBe(2);
var clients = factory.Clients.Cast<OrderingClient>().ToList();
// Every client really served reads (2 on A, 1 on B), so "peak 1" is measured, not vacuous …
clients.Select(c => c.Reads).OrderBy(n => n).ShouldBe([1, 2]);
clients.ShouldAllBe(c => c.PeakInFlight == 1,
"one device's socket is still request→response: never two reads in flight on it");
// … and the two devices overlapped: two reads were in flight at once across the clients.
global[1].ShouldBe(2, "the devices must be read at the same time, not one after the other");
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A device that accepted TCP and then stopped answering: every read hangs until the call timeout.</summary>
private sealed class StalledClient : FakeFocasClient
{
public int Reads;
public override async Task<(object? value, uint status)> ReadAsync(
FocasAddress address, FocasDataType type, CancellationToken ct)
{
Interlocked.Increment(ref Reads);
await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
return (null, FocasStatusMapper.Good);
}
}
/// <summary>
/// #740 review I1 — <c>Task.WhenAll</c> waits for the slowest device, so concurrency alone still made a
/// stalled device cost (its tag count × the call timeout) per batch, and the healthy device's values waited
/// that long. After the first timeout on a device its remaining tags in the batch fail fast: the batch is
/// bounded by about ONE call timeout and the healthy device's values are Good.
/// </summary>
[Fact]
public async Task A_stalled_device_costs_one_call_timeout_per_batch_not_one_per_tag()
{
var callTimeout = TimeSpan.FromMilliseconds(200);
var factory = new FakeFocasClientFactory();
factory.Customise = () => factory.Clients.Count == 0 ? new StalledClient() : new FakeFocasClient { Values = { ["R100"] = (sbyte)9 } };
var stalledTags = Enumerable.Range(0, 10)
.Select(i => FocasRawTags.Entry(new FocasTagDefinition($"raw/A{i}", "A", $"R{100 + i}", FocasDataType.Byte)));
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(HostA, DeviceName: "A"), new FocasDeviceOptions(HostB, DeviceName: "B")],
RawTags = [.. stalledTags, FocasRawTags.Entry(new FocasTagDefinition("raw/B1", "B", "R100", FocasDataType.Byte))],
Probe = new FocasProbeOptions { Enabled = false },
PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = false },
Timeout = callTimeout,
}, "focas-stall", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
// Connect device A first so it gets the stalled client (connect order = client order).
await drv.ReadAsync(["raw/A0"], CancellationToken.None);
var refs = Enumerable.Range(0, 10).Select(i => $"raw/A{i}").Append("raw/B1").ToArray();
var sw = System.Diagnostics.Stopwatch.StartNew();
var snaps = await drv.ReadAsync(refs, CancellationToken.None);
sw.Stop();
sw.Elapsed.ShouldBeLessThan(callTimeout * 3, "one stalled device must cost ~one call timeout, not ten");
snaps[10].StatusCode.ShouldBe(FocasStatusMapper.Good);
snaps[10].Value.ShouldBe((sbyte)9);
snaps.Take(10).ShouldAllBe(s => s.StatusCode == FocasStatusMapper.BadCommunicationError);
((StalledClient)factory.Clients[0]).Reads.ShouldBe(2, "one read in the warm-up batch, one in the measured batch");
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>Counts per-tag reads; one address can be made to time out (throw) or answer a status.</summary>
private sealed class CountingClient : FakeFocasClient
{
public int Reads;
public string? TimeoutAt;
public override async Task<(object? value, uint status)> ReadAsync(
FocasAddress address, FocasDataType type, CancellationToken ct)
{
Interlocked.Increment(ref Reads);
if (address.Canonical == TimeoutAt)
{
await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
}
return await base.ReadAsync(address, type, ct).ConfigureAwait(false);
}
}
private static FocasDriver OneDevice(FakeFocasClientFactory factory, bool coalesce, params string[] addresses) => new(
new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(HostA, DeviceName: "A")],
RawTags = [.. addresses.Select(a => FocasRawTags.Entry(new FocasTagDefinition($"raw/{a}", "A", a, FocasDataType.Byte)))],
Probe = new FocasProbeOptions { Enabled = false },
PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = coalesce },
Timeout = TimeSpan.FromMilliseconds(200),
}, "focas-linkdown", factory);
/// <summary>
/// #740 review round 2 N1 — a status the CNC RETURNED (EW_REJECT and every unlisted rc map to
/// BadCommunicationError) proves the device answered, so it must not end the device's batch: the tags after it
/// still read. Before the fix, a deterministic rc on the first tag starved every later tag forever.
/// </summary>
[Fact]
public async Task A_returned_comms_status_does_not_fail_the_rest_of_the_batch()
{
var client = new CountingClient { Values = { ["R101"] = (sbyte)1, ["R102"] = (sbyte)2 } };
client.ReadStatuses["R100"] = FocasStatusMapper.MapFocasReturn(13); // EW_REJECT ⇒ BadCommunicationError
var drv = OneDevice(new FakeFocasClientFactory { Customise = () => client }, coalesce: false, "R100", "R101", "R102");
await drv.InitializeAsync("{}", CancellationToken.None);
var snaps = await drv.ReadAsync(["raw/R100", "raw/R101", "raw/R102"], CancellationToken.None);
snaps[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
snaps[1].StatusCode.ShouldBe(FocasStatusMapper.Good);
snaps[2].Value.ShouldBe((sbyte)2);
client.Reads.ShouldBe(3, "every tag was really read");
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>Positive control for N1: a TIMEOUT (no reply at all) still fails the rest of the batch without reading it.</summary>
[Fact]
public async Task A_timeout_still_fails_the_rest_of_the_batch_fast()
{
var client = new CountingClient { TimeoutAt = "R100", Values = { ["R101"] = (sbyte)1 } };
var drv = OneDevice(new FakeFocasClientFactory { Customise = () => client }, coalesce: false, "R100", "R101", "R102");
await drv.InitializeAsync("{}", CancellationToken.None);
var snaps = await drv.ReadAsync(["raw/R100", "raw/R101", "raw/R102"], CancellationToken.None);
snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.BadCommunicationError);
client.Reads.ShouldBe(1, "only the tag that timed out touched the wire");
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>
/// Review round 2 m5 — the coalescer's link-down paths: a transport fault on the first merged range fails the
/// second range WITHOUT a second range read, and the per-tag tags of the same device without a per-tag read.
/// </summary>
[Fact]
public async Task Coalescer_transport_fault_fails_later_ranges_and_per_tag_tags_without_reading_them()
{
var client = new CountingClient { ThrowOnPmcRange = true };
// Two ranges far apart (gap > MaxGapBytes) + one lone tag that takes the per-tag path.
var drv = OneDevice(new FakeFocasClientFactory { Customise = () => client }, coalesce: true,
"R100", "R101", "R500", "R501", "D900");
await drv.InitializeAsync("{}", CancellationToken.None);
var snaps = await drv.ReadAsync(["raw/R100", "raw/R101", "raw/R500", "raw/R501", "raw/D900"], CancellationToken.None);
snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.BadCommunicationError);
client.PmcRangeReads.Count.ShouldBe(1, "the second range must not pay another transport attempt");
client.Reads.ShouldBe(0, "the per-tag tag must not pay one either");
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>Review round 2 m5 — a coalescer CONNECT failure fails the per-tag tags of the device without a second connect.</summary>
[Fact]
public async Task Coalescer_connect_failure_fails_per_tag_tags_without_reconnecting()
{
var factory = new FakeFocasClientFactory { Customise = () => new CountingClient { ThrowOnConnect = true } };
var drv = OneDevice(factory, coalesce: true, "R100", "R101", "D900");
await drv.InitializeAsync("{}", CancellationToken.None);
var snaps = await drv.ReadAsync(["raw/R100", "raw/R101", "raw/D900"], CancellationToken.None);
snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.BadCommunicationError);
factory.Clients.Count.ShouldBe(1, "one connect attempt per device per batch");
factory.Clients.Sum(c => c.ConnectCount).ShouldBe(1);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>Results land at their own indices whatever order the devices finish in.</summary>
[Fact]
public async Task Results_keep_their_request_order_across_devices()
{
var factory = new FakeFocasClientFactory();
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(HostA, DeviceName: "A"), new FocasDeviceOptions(HostB, DeviceName: "B")],
RawTags =
[
FocasRawTags.Entry(new FocasTagDefinition("raw/A1", "A", "R100", FocasDataType.Byte)),
FocasRawTags.Entry(new FocasTagDefinition("raw/B1", "B", "R200", FocasDataType.Byte)),
],
Probe = new FocasProbeOptions { Enabled = false },
}, "focas-p9-order", factory)
;
factory.Customise = () => new FakeFocasClient { Values = { ["R100"] = (sbyte)1, ["R200"] = (sbyte)2 } };
await drv.InitializeAsync("{}", CancellationToken.None);
var snaps = await drv.ReadAsync(["raw/B1", "nope", "raw/A1"], CancellationToken.None);
snaps[0].Value.ShouldBe((sbyte)2);
snaps[1].StatusCode.ShouldBe(FocasStatusMapper.BadNodeIdUnknown);
snaps[2].Value.ShouldBe((sbyte)1);
await drv.ShutdownAsync(CancellationToken.None);
}
}
@@ -0,0 +1,200 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// #740 / R07:H13 — the device table is read by the retained poll engine, the fixed-tree loop, the alarm
/// loop and the actor's health tick, so a re-initialise must build a fresh table and swap it in rather than
/// clearing and refilling the live one. Also pins the lifecycle overlaps the runtime produces since Lane A:
/// a dispose running beside a still-running shutdown, and an abandoned initialise overlapping the next
/// attempt's shutdown.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasDeviceTableLifecycleTests
{
private const string Host = "focas://10.0.0.5:8193";
private static FocasDriverOptions Options(bool probe = false, bool fixedTree = false) => new()
{
Devices = [new FocasDeviceOptions(Host)],
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/R", Host, "R100", FocasDataType.Byte))],
Probe = new FocasProbeOptions { Enabled = probe, Interval = TimeSpan.FromMilliseconds(20) },
FixedTree = new FocasFixedTreeOptions { Enabled = fixedTree, PollInterval = TimeSpan.FromMilliseconds(10) },
};
/// <summary>A re-initialise publishes a NEW table; a reader holding the old one never sees it emptied.</summary>
[Fact]
public async Task Reinitialize_swaps_in_a_new_table_and_never_mutates_the_one_a_reader_holds()
{
var drv = new FocasDriver(Options(), "focas-swap", new FakeFocasClientFactory());
await drv.InitializeAsync("{}", CancellationToken.None);
var before = drv.DeviceTable;
before.Count.ShouldBe(1);
await drv.InitializeAsync("{}", CancellationToken.None);
drv.DeviceTable.ShouldNotBeSameAs(before);
drv.DeviceTable.Count.ShouldBe(1);
before.Count.ShouldBe(1, "the table a concurrent reader enumerates must not be cleared under it");
await drv.ShutdownAsync(CancellationToken.None);
before.Count.ShouldBe(1);
drv.DeviceTable.Count.ShouldBe(0);
}
/// <summary>Hammering every table reader during repeated re-initialises throws nothing — and reads still succeed.</summary>
[Fact]
public async Task Readers_survive_repeated_reinitialise()
{
var factory = new FakeFocasClientFactory();
var drv = new FocasDriver(Options(fixedTree: true), "focas-hammer", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
using var stop = new CancellationTokenSource();
var good = 0;
var errors = new List<Exception>();
var reader = Task.Run(async () =>
{
while (!stop.IsCancellationRequested)
{
try
{
_ = drv.GetHostStatuses();
var snaps = await drv.ReadAsync(["raw/R", $"{Host}/Identity/SeriesNumber"], CancellationToken.None);
if (snaps[0].StatusCode == FocasStatusMapper.Good) Interlocked.Increment(ref good);
}
catch (Exception ex) { lock (errors) errors.Add(ex); }
}
});
for (var i = 0; i < 100; i++)
await drv.InitializeAsync("{}", CancellationToken.None);
await Task.Delay(100);
stop.Cancel();
await reader;
await drv.ShutdownAsync(CancellationToken.None);
errors.ShouldBeEmpty();
good.ShouldBeGreaterThan(0, "positive control: the reader really did read through the table");
}
/// <summary>A client whose connect parks until released and ignores cancellation.</summary>
private sealed class ParkedConnectClient(TaskCompletionSource entered, Task release) : FakeFocasClient
{
public override async Task ConnectAsync(FocasHostAddress address, TimeSpan timeout, CancellationToken ct)
{
entered.TrySetResult();
await release.ConfigureAwait(false);
await base.ConnectAsync(address, timeout, CancellationToken.None).ConfigureAwait(false);
}
}
/// <summary>
/// A data-path read parked in connect when shutdown runs must not leave a live client on the released
/// device: the device is retired, so the connect that completes afterwards disposes what it opened.
/// </summary>
[Fact]
public async Task Connect_completing_after_shutdown_does_not_leave_a_live_client()
{
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var factory = new FakeFocasClientFactory { Customise = () => new ParkedConnectClient(entered, release.Task) };
var drv = new FocasDriver(Options(), "focas-retire", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
var read = drv.ReadAsync(["raw/R"], CancellationToken.None);
await entered.Task.WaitAsync(TimeSpan.FromSeconds(5));
var shutdown = drv.ShutdownAsync(CancellationToken.None);
await Task.Delay(50);
release.SetResult();
await shutdown.WaitAsync(TimeSpan.FromSeconds(10));
var snaps = await read.WaitAsync(TimeSpan.FromSeconds(5));
factory.Clients.Count.ShouldBe(1);
factory.Clients[0].IsConnected.ShouldBeFalse("a connect finishing on a retired device must dispose its client");
factory.Clients[0].DisposeCount.ShouldBeGreaterThanOrEqualTo(1);
snaps[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
}
/// <summary>
/// An initialise abandoned mid-way (the runtime timed it out and moved on to a shutdown) must not
/// resurrect the driver when it finally completes: no device table, no loops, no connection.
/// </summary>
[Fact]
public async Task Abandoned_initialise_finishing_after_shutdown_starts_nothing()
{
var factory = new FakeFocasClientFactory();
var options = Options(probe: true);
using var gate = new ManualResetEventSlim(false);
var inRebind = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var drv = new FocasDriver(options, "focas-abandoned", factory, rebind: _ =>
{
inRebind.TrySetResult();
gate.Wait(TimeSpan.FromSeconds(10));
return new FocasDriverBinding(options, factory);
});
var init = Task.Run(() => drv.InitializeAsync("{\"devices\":[]}", CancellationToken.None));
await inRebind.Task.WaitAsync(TimeSpan.FromSeconds(5));
await drv.ShutdownAsync(CancellationToken.None);
gate.Set();
await Should.ThrowAsync<OperationCanceledException>(() => init.WaitAsync(TimeSpan.FromSeconds(10)));
await Task.Delay(200); // several probe intervals — a started loop would have connected by now
drv.DeviceCount.ShouldBe(0);
factory.Clients.ShouldBeEmpty();
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
// Positive control: the same driver, initialised without being superseded, does start its probe.
await drv.InitializeAsync("{\"devices\":[]}", CancellationToken.None);
await WaitUntil(() => factory.Clients.Count > 0);
drv.DeviceCount.ShouldBe(1);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
await drv.DisposeAsync();
}
/// <summary>A loop wedged in a read that ignores cancellation for a while.</summary>
private sealed class SlowProbeClient : FakeFocasClient
{
public override async Task<bool> ProbeAsync(CancellationToken ct)
{
await Task.Delay(400, CancellationToken.None).ConfigureAwait(false);
return true;
}
}
/// <summary>
/// Lane A disposes while its own shutdown may still be running. The dispose must not return before the
/// teardown it overlaps has released the client — it waits for it rather than finding an empty table.
/// </summary>
[Fact]
public async Task Dispose_concurrent_with_shutdown_waits_for_the_teardown()
{
var factory = new FakeFocasClientFactory { Customise = () => new SlowProbeClient() };
var drv = new FocasDriver(Options(probe: true), "focas-dispose", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
await WaitUntil(() => factory.Clients.Count > 0 && factory.Clients[0].IsConnected);
var shutdown = drv.ShutdownAsync(CancellationToken.None);
var dispose = drv.DisposeAsync().AsTask();
await dispose.WaitAsync(TimeSpan.FromSeconds(10));
factory.Clients.ShouldAllBe(c => !c.IsConnected);
factory.Clients.ShouldAllBe(c => c.DisposeCount >= 1);
await shutdown.WaitAsync(TimeSpan.FromSeconds(10));
drv.DeviceCount.ShouldBe(0);
}
private static async Task WaitUntil(Func<bool> condition)
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5);
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
await Task.Delay(10);
}
}
}
@@ -0,0 +1,163 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// #740 / R07:H15 — the fixed-tree loop swallowed every failure at Debug and never touched driver health, so a
/// driver serving only <c>FIXED:</c> tags (Identity / Axes / Program / Status — the typical FOCAS shape) read
/// Healthy on <c>/hosts</c> with every node Bad, and the #527 comms-loss watchdog could never fire. The loop
/// now credits each answered axis pass (Healthy + LastSuccessfulRead) and, once a device has been silent past
/// the fixed-tree staleness budget, degrades the driver with LastSuccessfulRead frozen.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasFixedTreeHealthTests
{
private const string Host = "focas://10.0.0.5:8193";
private static readonly TimeSpan Stale = TimeSpan.FromMilliseconds(300);
private static readonly TimeSpan BlipBudget = TimeSpan.FromMilliseconds(1500);
/// <summary>A CNC that can be switched off (transport failure) or made to refuse the dynamic read.</summary>
private sealed class CncSwitch
{
public volatile bool Down;
public volatile bool Refuse;
/// <summary>Transport failures served so far — what the tests wait on instead of wall-clock delays.</summary>
public int Failures;
/// <summary>Refusals served so far.</summary>
public int Refusals;
}
private sealed class SwitchableCnc(CncSwitch sw) : FakeFocasClient
{
private bool Down => sw.Down;
private bool Refuse => sw.Refuse;
public override Task<FocasSysInfo> GetSysInfoAsync(CancellationToken ct)
{
if (!Down) return base.GetSysInfoAsync(ct);
Interlocked.Increment(ref sw.Failures);
throw new IOException("socket closed");
}
public override Task<FocasDynamicSnapshot> ReadDynamicAsync(int axisIndex, CancellationToken ct)
{
if (Down)
{
Interlocked.Increment(ref sw.Failures);
throw new IOException("socket closed");
}
if (Refuse)
{
Interlocked.Increment(ref sw.Refusals);
throw new FocasWireException("cnc_rddynamic2 refused", rc: 6);
}
return base.ReadDynamicAsync(axisIndex, ct);
}
}
private static FocasDriver FixedOnlyDriver(CncSwitch sw, TimeSpan? budget = null) => new(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/X", Host, "FIXED:Axes/X/AbsolutePosition", FocasDataType.Float64))],
Probe = new FocasProbeOptions { Enabled = false },
FixedTree = new FocasFixedTreeOptions
{
PollInterval = TimeSpan.FromMilliseconds(20),
ProgramPollInterval = TimeSpan.Zero,
TimerPollInterval = TimeSpan.Zero,
},
}, "focas-fixed-health", new FakeFocasClientFactory { Customise = () => new SwitchableCnc(sw) })
{
FixedTreeStaleBudgetOverride = budget ?? Stale,
};
/// <summary>
/// Positive control first, then the outage: an answering CNC keeps the driver Healthy with a moving
/// LastSuccessfulRead; a CNC that stops answering degrades it once the budget passes, with
/// LastSuccessfulRead frozen (the watchdog's conjunction); and its return restores Healthy.
/// </summary>
[Fact]
public async Task Fixed_only_driver_degrades_when_its_cnc_disappears_and_recovers_when_it_returns()
{
var sw = new CncSwitch();
// A wide budget (review round 2 m4): two 20 ms ticks always land well inside it, so the "blip does not
// degrade" absence check below runs unconditionally instead of being skipped on a slow agent.
await using var drv = FixedOnlyDriver(sw, BlipBudget);
await drv.InitializeAsync("{}", CancellationToken.None);
await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null);
var first = drv.GetHealth().LastSuccessfulRead!.Value;
// An answered axis pass is a successful read: wait for it to move rather than sleeping.
await WaitUntil(() => drv.GetHealth().LastSuccessfulRead > first);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
sw.Down = true;
var lastAnswer = drv.GetDeviceState(Host)!.LastFixedPollUtc!.Value;
// Two failed ticks have demonstrably happened (positive evidence for the absence check below) …
await WaitUntil(() => Volatile.Read(ref sw.Failures) >= 2);
// … inside the budget (asserted, not assumed), and the driver is still Healthy.
(DateTime.UtcNow - lastAnswer).ShouldBeLessThan(BlipBudget, "the two failed ticks must land inside the budget");
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade");
await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded);
var frozen = drv.GetHealth().LastSuccessfulRead;
var failuresAtDegrade = Volatile.Read(ref sw.Failures);
await WaitUntil(() => Volatile.Read(ref sw.Failures) >= failuresAtDegrade + 5);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "LastSuccessfulRead must stay frozen while the CNC is silent");
sw.Down = false;
await WaitUntil(() => drv.GetHealth().State == DriverState.Healthy);
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(frozen!.Value);
}
/// <summary>A CNC that is down from the start (bootstrap never succeeds) degrades once the budget passes.</summary>
[Fact]
public async Task Bootstrap_that_never_succeeds_degrades_the_driver()
{
var sw = new CncSwitch { Down = true };
await using var drv = FixedOnlyDriver(sw);
await drv.InitializeAsync("{}", CancellationToken.None);
// The bootstrap retries every 2 s, so the first failure is what has to trip it.
await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(6));
sw.Down = false;
}
/// <summary>
/// A CNC that ANSWERS with a refusal is not a comms loss: the transport is fine, so the driver is not
/// degraded (else the watchdog would reconnect-loop a production control).
/// </summary>
[Fact]
public async Task Cnc_refusal_of_the_dynamic_read_does_not_degrade()
{
var sw = new CncSwitch();
await using var drv = FixedOnlyDriver(sw);
await drv.InitializeAsync("{}", CancellationToken.None);
await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null);
sw.Refuse = true;
var started = DateTime.UtcNow;
await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 1);
var atFirstRefusal = drv.GetHealth().LastSuccessfulRead!.Value;
// Refused ticks really happened, for longer than the staleness budget …
await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 5 && DateTime.UtcNow - started > Stale * 2);
// … and the driver stayed Healthy, with each refusal crediting the answer (review minor 2).
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(atFirstRefusal,
"a refusal is an answer from the CNC — LastSuccessfulRead advances");
sw.Refuse = false;
}
private static async Task WaitUntil(Func<bool> condition, TimeSpan? timeout = null)
{
var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(5));
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
await Task.Delay(10);
}
}
}
@@ -62,6 +62,73 @@ public sealed class FocasHandleRecycleTests
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A client whose reads take a while — long enough for a recycle tick to land mid-read.</summary>
private sealed class SlowReadClient : FakeFocasClient
{
public override async Task<(object? value, uint status)> ReadAsync(
FocasAddress address, FocasDataType type, CancellationToken ct)
{
await Task.Delay(200, ct).ConfigureAwait(false);
if (!IsConnected) throw new ObjectDisposedException(nameof(SlowReadClient), "disposed mid-read");
return ((sbyte)5, FocasStatusMapper.Good);
}
}
/// <summary>
/// #740 / R07:L6 — the recycle loop disposed the device client with no gate, so a recycle tick landing
/// mid-read killed that read (BadCommunicationError + Degraded, every interval). The recycle now takes the
/// device's connect gate and lets the in-flight read finish before disposing the client it detached.
/// </summary>
[Fact]
public async Task Recycle_tick_landing_mid_read_lets_the_read_finish()
{
var factory = new FakeFocasClientFactory { Customise = () => new SlowReadClient() };
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions("focas://10.0.0.5:8193")],
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("R", "focas://10.0.0.5:8193", "R100", FocasDataType.Byte))],
Probe = new FocasProbeOptions { Enabled = false },
HandleRecycle = new FocasHandleRecycleOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(50) },
}, "drv-1", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
// Each 200 ms read spans at least three recycle ticks.
for (var i = 0; i < 3; i++)
{
var snap = (await drv.ReadAsync(["R"], CancellationToken.None))[0];
snap.StatusCode.ShouldBe(FocasStatusMapper.Good, $"read {i} was killed by a recycle tick");
snap.Value.ShouldBe((sbyte)5);
}
drv.GetHealth().State.ShouldBe(Core.Abstractions.DriverState.Healthy);
// Positive control: the recycle really ran — the first client was disposed and a fresh one opened.
await WaitFor(() => factory.Clients[0].DisposeCount > 0, TimeSpan.FromSeconds(3));
factory.Clients[0].DisposeCount.ShouldBeGreaterThan(0);
factory.Clients.Count.ShouldBeGreaterThan(1);
await drv.ShutdownAsync(CancellationToken.None);
factory.Clients.ShouldAllBe(c => !c.IsConnected, "no client opened around a recycle may be orphaned");
}
/// <summary>DisposeWhenIdleAsync drains the in-flight op, then disposes; a later op is refused, not run on a dead socket.</summary>
[Fact]
public async Task DisposeWhenIdle_drains_the_in_flight_op_then_refuses_new_ones()
{
var inner = new SlowReadClient();
await inner.ConnectAsync(new FocasHostAddress("10.0.0.5", 8193), TimeSpan.FromSeconds(1), CancellationToken.None);
var sync = new SynchronizedFocasClient(inner, TimeSpan.FromSeconds(5));
var addr = FocasAddress.TryParse("R100")!;
var read = sync.ReadAsync(addr, FocasDataType.Byte, CancellationToken.None);
await Task.Delay(20);
sync.PendingOperations.ShouldBe(1);
var dispose = sync.DisposeWhenIdleAsync(TimeSpan.FromSeconds(5));
(await read).status.ShouldBe(FocasStatusMapper.Good);
await dispose;
inner.DisposeCount.ShouldBe(1);
await Should.ThrowAsync<ObjectDisposedException>(() => sync.ReadAsync(addr, FocasDataType.Byte, CancellationToken.None));
}
private static async Task WaitFor(Func<bool> pred, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
@@ -57,4 +57,32 @@ public sealed class FocasPollErrorHealthTests
drv.GetHealth().State.ShouldBe(DriverState.Faulted);
}
/// <summary>
/// #740 review round 2 m3 — a poll failure carries the generation its batch STARTED under, so a batch that
/// began before a re-initialise cannot degrade the new generation. Positive control: a failure stamped with
/// the live generation (or unstamped) does degrade.
/// </summary>
[Fact]
public async Task PollReaderFailure_from_a_previous_generation_does_not_degrade_the_new_one()
{
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions("focas://10.0.0.5:8193")],
Probe = new FocasProbeOptions { Enabled = false },
}, "focas-1", new FakeFocasClientFactory());
await drv.InitializeAsync("{}", CancellationToken.None);
var oldGeneration = drv.LifecycleGeneration;
await drv.InitializeAsync("{}", CancellationToken.None);
var stale = new InvalidOperationException("old batch boom");
stale.Data[FocasDriver.BatchGenerationKey] = oldGeneration;
drv.HandlePollError(stale);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
var live = new InvalidOperationException("live batch boom");
live.Data[FocasDriver.BatchGenerationKey] = drv.LifecycleGeneration;
drv.HandlePollError(live);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
}
}
@@ -0,0 +1,212 @@
using Microsoft.Extensions.Logging;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// #740 driver-lane health checklist (drivers.md, #527/#611): a deterministic CNC refusal — the control
/// answered with a non-OK return code, or with a value the tag's type cannot hold (vacant macro, fractional
/// macro on an integer tag) — is per-tag Bad with ONE Warning per tag and NO health write, and it still
/// proves the device answered (LastSuccessfulRead advances). Only a comms-class status degrades the driver.
/// Before this, every non-Good status latched Degraded every poll, so one vacant macro made a reachable CNC
/// look sick forever.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasReadHealthContractTests
{
private const string Host = "focas://10.0.0.5:8193";
private sealed class WarningCounter : ILogger<FocasDriver>
{
private int _warnings;
public int Warnings => Volatile.Read(ref _warnings);
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
Func<TState, Exception?, string> formatter)
{
if (logLevel == LogLevel.Warning && formatter(state, exception).Contains("refused", StringComparison.Ordinal))
Interlocked.Increment(ref _warnings);
}
}
private static (FocasDriver drv, FakeFocasClient client, WarningCounter log) NewDriver(uint r100Status)
{
var client = new FakeFocasClient { Values = { ["R101"] = (sbyte)1 } };
client.ReadStatuses["R100"] = r100Status;
var log = new WarningCounter();
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags =
[
FocasRawTags.Entry(new FocasTagDefinition("raw/R100", Host, "R100", FocasDataType.Byte)),
FocasRawTags.Entry(new FocasTagDefinition("raw/R101", Host, "R101", FocasDataType.Byte)),
],
Probe = new FocasProbeOptions { Enabled = false },
PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = false },
}, "focas-contract", new FakeFocasClientFactory { Customise = () => client }, log);
return (drv, client, log);
}
/// <summary>A refusal leaves health alone, warns once, and still credits the answer.</summary>
[Theory]
[InlineData(FocasStatusMapper.BadOutOfRange)] // EW_NUMBER
[InlineData(FocasStatusMapper.BadNotSupported)] // EW_FUNC / EW_NOOPT
[InlineData(FocasStatusMapper.BadNoData)] // vacant macro
[InlineData(FocasStatusMapper.BadTypeMismatch)] // fractional macro on an integer tag
[InlineData(FocasStatusMapper.BadDeviceFailure)] // EW_BUSY
public async Task Cnc_refusal_is_per_tag_bad_with_one_warning_and_no_health_write(uint refusal)
{
var (drv, _, log) = NewDriver(refusal);
await drv.InitializeAsync("{}", CancellationToken.None);
var atInit = drv.GetHealth().LastSuccessfulRead;
for (var i = 0; i < 3; i++)
{
await Task.Delay(5);
var snap = (await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0];
snap.StatusCode.ShouldBe(refusal);
}
var health = drv.GetHealth();
health.State.ShouldBe(DriverState.Healthy, "a CNC refusal is not a comms fault");
health.LastSuccessfulRead!.Value.ShouldBeGreaterThan(atInit!.Value, "the CNC answered — that is evidence of life");
log.Warnings.ShouldBe(1, "one Warning per refused tag, not one per poll");
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A refused tag that recovers and is refused again warns again (the dedup clears on Good).</summary>
[Fact]
public async Task Refusal_warning_rearms_after_the_tag_reads_good()
{
var (drv, client, log) = NewDriver(FocasStatusMapper.BadOutOfRange);
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
client.ReadStatuses.Remove("R100");
client.Values["R100"] = (sbyte)3;
(await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0].StatusCode.ShouldBe(FocasStatusMapper.Good);
client.ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
log.Warnings.ShouldBe(2);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>Positive control: a comms-class status still degrades, preserving LastSuccessfulRead.</summary>
[Fact]
public async Task Comms_status_still_degrades_with_last_successful_read_preserved()
{
var (drv, _, _) = NewDriver(FocasStatusMapper.BadCommunicationError);
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.ReadAsync(["raw/R101"], CancellationToken.None);
var lastGood = drv.GetHealth().LastSuccessfulRead;
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
var health = drv.GetHealth();
health.State.ShouldBe(DriverState.Degraded);
health.LastSuccessfulRead.ShouldBe(lastGood);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>
/// Review minor 2 — a device that recovers from a comms fault and then only REFUSES (every tag it serves is
/// refused) is answering, so it is restored to Healthy instead of staying Degraded forever.
/// </summary>
[Fact]
public async Task Refusal_after_a_comms_fault_restores_healthy()
{
var (drv, client, _) = NewDriver(FocasStatusMapper.BadCommunicationError);
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
drv.GetHealth().State.ShouldBe(DriverState.Degraded, "positive control: the comms fault degraded it");
client.ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
await drv.ReadAsync(["raw/R100"], CancellationToken.None);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A read whose CNC answer is held until released, then fails at the transport.</summary>
private sealed class HeldReadClient(TaskCompletionSource entered, Task release) : FakeFocasClient
{
public volatile bool Hold = true;
public override async Task<(object? value, uint status)> ReadAsync(
FocasAddress address, FocasDataType type, CancellationToken ct)
{
if (!Hold) return await base.ReadAsync(address, type, ct).ConfigureAwait(false);
entered.TrySetResult();
await release.ConfigureAwait(false);
throw new IOException("socket reset");
}
}
/// <summary>
/// Review minor 8 — a read that started against the PREVIOUS table and fails after a re-initialise must not
/// write its Degraded onto the new generation's health. Positive control: the same failure under the live
/// generation does degrade.
/// </summary>
[Fact]
public async Task A_read_from_the_previous_generation_cannot_degrade_the_new_one()
{
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var clients = new List<HeldReadClient>();
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/R100", Host, "R100", FocasDataType.Byte))],
Probe = new FocasProbeOptions { Enabled = false },
Timeout = TimeSpan.FromSeconds(10),
}, "focas-gen", new FakeFocasClientFactory
{
Customise = () => { var c = new HeldReadClient(entered, release.Task); lock (clients) clients.Add(c); return c; },
});
await drv.InitializeAsync("{}", CancellationToken.None);
var stale = drv.ReadAsync(["raw/R100"], CancellationToken.None);
await entered.Task.WaitAsync(TimeSpan.FromSeconds(5));
await drv.InitializeAsync("{}", CancellationToken.None); // a new generation, Healthy
release.SetResult();
(await stale.WaitAsync(TimeSpan.FromSeconds(5)))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "the old generation's failure must not land on the new one");
// Positive control: a failure under the live generation degrades.
var live = drv.ReadAsync(["raw/R100"], CancellationToken.None);
(await live.WaitAsync(TimeSpan.FromSeconds(5)))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
await drv.ShutdownAsync(CancellationToken.None);
}
/// <summary>A write the driver refuses (NotSupportedException) is not a comms fault either.</summary>
[Fact]
public async Task Write_refusal_does_not_degrade()
{
var client = new ThrowingWriteClient();
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/W", Host, "R100", FocasDataType.Byte, Writable: true))],
Probe = new FocasProbeOptions { Enabled = false },
}, "focas-write-refusal", new FakeFocasClientFactory { Customise = () => client });
await drv.InitializeAsync("{}", CancellationToken.None);
var result = await drv.WriteAsync([new WriteRequest("raw/W", (sbyte)1)], CancellationToken.None);
result[0].StatusCode.ShouldBe(FocasStatusMapper.BadNotSupported);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
await drv.ShutdownAsync(CancellationToken.None);
}
private sealed class ThrowingWriteClient : FakeFocasClient
{
public override Task<uint> WriteAsync(FocasAddress address, FocasDataType type, object? value, CancellationToken ct) =>
throw new NotSupportedException("bit writes are not supported on this area");
}
}
@@ -0,0 +1,104 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// #740 / R07:U8 — <see cref="FocasDataType.String"/> was authorable on any address, but no area read
/// produces a string: PMC fell through to a byte read, PARAM to an Int32, MACRO to a double, and a
/// String-typed node received a boxed number at Good. String is now refused on PMC / PARAM / MACRO by the
/// one shared check, everywhere a tag is authored or loaded; the fixed tree (Identity / Program name, …),
/// whose values really are strings, keeps it.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasStringDataTypeTests
{
private const string Host = "focas://10.0.0.5:8193";
private static string Config(string address, string dataType) =>
$$"""{"address":"{{address}}","dataType":"{{dataType}}"}""";
/// <summary>String on an area read is refused by the mapper.</summary>
[Theory]
[InlineData("R100")]
[InlineData("X0.0")]
[InlineData("PARAM:6711")]
[InlineData("MACRO:500")]
public void Mapper_refuses_string_on_an_area_read(string address)
{
FocasTagDefinitionFactory.FromTagConfig(Config(address, "String"), "raw/T", out _).ShouldBeFalse();
FocasTagDefinitionFactory.DescribeDataTypeMismatch(FocasDataType.String, address).ShouldNotBeNull();
}
/// <summary>Positive control: numeric types on the same addresses, and String on the fixed tree, still map.</summary>
[Theory]
[InlineData("R100", "Byte")]
[InlineData("PARAM:6711", "Int32")]
[InlineData("MACRO:500", "Float64")]
[InlineData("FIXED:Identity/SeriesNumber", "String")]
[InlineData("FIXED:Program/Name", "String")]
public void Mapper_accepts_numeric_area_reads_and_string_fixed_tree_paths(string address, string dataType)
{
FocasTagDefinitionFactory.FromTagConfig(Config(address, dataType), "raw/T", out var def).ShouldBeTrue();
def.Address.ShouldBe(address);
}
/// <summary>An ALARM: tag has no value, so its data type is irrelevant and String is not refused.</summary>
[Fact]
public void Mapper_accepts_string_on_an_alarm_tag()
{
FocasTagDefinitionFactory.FromTagConfig(
"""{"address":"ALARM:*","dataType":"String","alarm":{"severity":500}}""", "raw/A", out _).ShouldBeTrue();
}
/// <summary>An unparseable address is left to the address check — it still draws THAT diagnosis.</summary>
[Fact]
public void Unparseable_address_keeps_its_own_diagnosis()
{
FocasTagDefinitionFactory.DescribeDataTypeMismatch(FocasDataType.String, "not-an-address").ShouldBeNull();
FocasTagDefinitionFactory.FromTagConfig(Config("not-an-address", "String"), "raw/T", out _).ShouldBeTrue();
}
/// <summary>The deploy-time inspector names the reason rather than falling to the generic backstop.</summary>
[Fact]
public void Inspect_names_the_string_refusal()
{
var warnings = FocasTagDefinitionFactory.Inspect(Config("MACRO:500", "String"));
warnings.ShouldContain(w => w.Contains("String", StringComparison.Ordinal) && w.Contains("MACRO:500", StringComparison.Ordinal));
FocasTagDefinitionFactory.Inspect(Config("MACRO:500", "Float64")).ShouldBeEmpty();
}
/// <summary>
/// At init a String area tag is skipped and reported (never served as a Good number); the numeric tag
/// beside it reads normally (positive control).
/// </summary>
[Fact]
public async Task Driver_skips_and_reports_a_string_area_tag()
{
var factory = new FakeFocasClientFactory
{
Customise = () => new FakeFocasClient { Values = { ["R100"] = (sbyte)7 } },
};
var drv = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host)],
RawTags =
[
new RawTagEntry("raw/Str", Config("R100", "String"), false, Host),
new RawTagEntry("raw/Num", Config("R100", "Byte"), false, Host),
],
Probe = new FocasProbeOptions { Enabled = false },
}, "focas-u8", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
var snaps = await drv.ReadAsync(["raw/Str", "raw/Num"], CancellationToken.None);
snaps[0].StatusCode.ShouldBe(FocasStatusMapper.BadNodeIdUnknown);
snaps[0].Value.ShouldBeNull();
snaps[1].StatusCode.ShouldBe(FocasStatusMapper.Good);
snaps[1].Value.ShouldBe((sbyte)7);
drv.GetSkippedTags()!.Sample.ShouldContain(s => s.RawPath == "raw/Str");
await drv.ShutdownAsync(CancellationToken.None);
}
}
@@ -0,0 +1,84 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire;
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests;
/// <summary>
/// #740 / R07:H14 — the capability matrix admits macro numbers up to 99999 (30i/31i/32i) and parameters up
/// to 29999 (Unknown: anything), and the FOCAS/2 request block carries 32-bit arguments, but the adapter
/// cast every number to <c>short</c>: <c>MACRO:99999</c> asked the CNC for <c>#-31073</c> and published
/// that variable Good under the authored tag. Numbers now travel as <c>int</c> end to end.
/// </summary>
[Trait("Category", "Unit")]
public sealed class FocasWideNumberTests
{
/// <summary>The macro request carries the full number in both 32-bit argument slots.</summary>
[Theory]
[InlineData(500)]
[InlineData(32767)]
[InlineData(32768)]
[InlineData(99999)]
public void Macro_request_carries_the_unnarrowed_number(int number)
{
var block = FocasWireClient.MacroRequest(number, pathId: 1);
block.Command.ShouldBe((ushort)0x0015);
block.Arg1.ShouldBe(number);
block.Arg2.ShouldBe(number);
}
/// <summary>
/// Positive control on the wire encoding: the request for #500 is byte-identical to the one the real
/// 31i-B answered in the <c>ref-macro-500-0x0015</c> capture, so widening changed nothing below 32768.
/// </summary>
[Fact]
public void Macro_request_for_500_matches_the_live_capture_bytes()
{
var body = FocasWireProtocol.BuildRequestBody([FocasWireClient.MacroRequest(500, pathId: 1)]);
Convert.ToHexString(body).ShouldBe("0001001C0001000100150000 01F4000001F4000000000000000000000000".Replace(" ", ""));
}
/// <summary>A global parameter puts its number in both arguments; an axis parameter puts the axis second.</summary>
[Theory]
[InlineData(6711, (short)0, 6711)]
[InlineData(40000, (short)0, 40000)]
[InlineData(40000, (short)2, 2)]
public void Parameter_request_carries_the_unnarrowed_number(int number, short axis, int expectedArg2)
{
var block = FocasWireClient.ParameterRequest(number, axis, pathId: 1);
block.Command.ShouldBe((ushort)0x000e);
block.Arg1.ShouldBe(number);
block.Arg2.ShouldBe(expectedArg2);
}
/// <summary>The decoded macro reports the number that was asked for, not a wrapped one.</summary>
[Fact]
public void Parsed_macro_keeps_a_number_above_short_range()
{
var payload = new byte[] { 0x05, 0xF5, 0xE1, 0x00, 0x00, 0x0A, 0x00, 0x08 };
var m = FocasWireClient.ParseMacro(99999, payload);
m.Number.ShouldBe(99999);
m.Scaled.ShouldBe(1.0);
}
/// <summary>
/// pmc_rdpmcrng addresses are 16-bit on the wire; a PMC number whose slot runs past 65535 is refused
/// rather than wrapped onto a different address (Unknown series admits any number).
/// </summary>
[Theory]
[InlineData(100, 1, true)]
[InlineData(65535, 1, true)]
[InlineData(65534, 2, true)]
[InlineData(65535, 2, false)]
[InlineData(70000, 1, false)]
public void Pmc_range_outside_16_bits_is_refused_not_wrapped(int number, int width, bool ok)
{
WireFocasClient.TryPmcRange(number, width, out var start, out var end).ShouldBe(ok);
if (ok)
{
((int)start).ShouldBe(number);
((int)end).ShouldBe(number + width - 1);
}
}
}
@@ -0,0 +1,146 @@
using System.Collections.Concurrent;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.IntegrationTests;
/// <summary>
/// Wire-level cases for the #739 High/Medium findings, against the live TwinCAT 3 XAR fixture
/// (skipped without it, like <see cref="TwinCAT3SmokeTests"/>). Every case that writes restores what it
/// changed: <c>GVL_Primitives.vWord</c> and <c>vString</c> are seeds the primitives theory reads.
/// </summary>
[Collection("TwinCATXar")]
[Trait("Category", "Integration")]
[Trait("Simulator", "TwinCAT-XAR")]
public sealed class TwinCATRemediationGateTests(TwinCATXarFixture sim)
{
private string Host => $"ads://{sim.TargetNetId}:{sim.AmsPort}";
private TwinCATDriver NewDriver(string id, bool native, params TwinCATTagDefinition[] tags) => new(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host, "XAR-VM")],
RawTags = TwinCATRawTags.Entries(tags),
UseNativeNotifications = native,
Timeout = TimeSpan.FromSeconds(5),
Probe = new TwinCATProbeOptions { Enabled = false },
}, id);
/// <summary>H11: a typo'd symbol in a native subscribe is isolated; the real counter keeps delivering.</summary>
[TwinCATFact]
public async Task Native_subscribe_isolates_an_unknown_symbol()
{
if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason);
await using var drv = NewDriver("tc3-gate-h11", native: true,
new TwinCATTagDefinition("Counter", Host, "GVL_Fixture.nCounter", TwinCATDataType.DInt),
new TwinCATTagDefinition("Typo", Host, "GVL_Fixture.nCounterTypo", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
var handle = await drv.SubscribeAsync(["Counter", "Typo"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5);
while (events.Count(e => e.FullReference == "Counter") < 2 && DateTime.UtcNow < deadline)
await Task.Delay(50, TestContext.Current.CancellationToken);
await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken);
events.Count(e => e.FullReference == "Counter" && e.Snapshot.StatusCode == 0u).ShouldBeGreaterThanOrEqualTo(2);
events.First(e => e.FullReference == "Typo").Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a refused symbol is not a comms fault");
}
/// <summary>H12: native pushes alone advance LastSuccessfulRead.</summary>
[TwinCATFact]
public async Task Native_pushes_advance_LastSuccessfulRead()
{
if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason);
await using var drv = NewDriver("tc3-gate-h12", native: true,
new TwinCATTagDefinition("Counter", Host, "GVL_Fixture.nCounter", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead!.Value;
var handle = await drv.SubscribeAsync(["Counter"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
await Task.Delay(1500, TestContext.Current.CancellationToken);
await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken);
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp.AddMilliseconds(500));
}
/// <summary>
/// H18: a bit write on the 16-bit <c>vWord</c> (0xBEEF) flips exactly that bit at the WORD's own width
/// (the old 32-bit UDInt RMW could not even address a WORD), and a bit beyond it is refused.
/// </summary>
[TwinCATFact]
public async Task Bit_write_on_a_WORD_flips_one_bit_and_refuses_bit_16()
{
if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason);
await using var drv = NewDriver("tc3-gate-h18", native: false,
new TwinCATTagDefinition("Word", Host, "GVL_Primitives.vWord", TwinCATDataType.UInt, Writable: false),
new TwinCATTagDefinition("Bit4", Host, "GVL_Primitives.vWord.4", TwinCATDataType.Bool),
new TwinCATTagDefinition("Bit16", Host, "GVL_Primitives.vWord.16", TwinCATDataType.Bool));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var before = Convert.ToUInt16((await drv.ReadAsync(["Word"], TestContext.Current.CancellationToken))[0].Value);
var bit4WasSet = (before & 0x10) != 0;
try
{
(await drv.WriteAsync([new WriteRequest("Bit4", !bit4WasSet)], TestContext.Current.CancellationToken))[0]
.StatusCode.ShouldBe(0u);
var after = Convert.ToUInt16((await drv.ReadAsync(["Word"], TestContext.Current.CancellationToken))[0].Value);
after.ShouldBe((ushort)(before ^ 0x10), "only bit 4 may change");
(await drv.WriteAsync([new WriteRequest("Bit16", true)], TestContext.Current.CancellationToken))[0]
.StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange);
}
finally
{
await drv.WriteAsync([new WriteRequest("Bit4", bit4WasSet)], CancellationToken.None);
}
Convert.ToUInt16((await drv.ReadAsync(["Word"], TestContext.Current.CancellationToken))[0].Value).ShouldBe(before);
}
/// <summary>U9: a whole-array write round-trips through the codec (CLR double[] converts to REAL elements).</summary>
[TwinCATFact]
public async Task Whole_array_write_round_trips()
{
if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason);
await using var drv = NewDriver("tc3-gate-u9-array", native: false,
new TwinCATTagDefinition("Arr", Host, "GVL_Arrays.aReal1D", TwinCATDataType.Real, ArrayLength: 32));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var values = Enumerable.Range(0, 32).Select(i => i * 1.5 + 0.25).ToArray();
(await drv.WriteAsync([new WriteRequest("Arr", values)], TestContext.Current.CancellationToken))[0]
.StatusCode.ShouldBe(0u);
var read = (await drv.ReadAsync(["Arr"], TestContext.Current.CancellationToken))[0];
read.StatusCode.ShouldBe(0u);
((float[])read.Value!).ShouldBe(values.Select(v => (float)v).ToArray());
(await drv.WriteAsync([new WriteRequest("Arr", values[..31])], TestContext.Current.CancellationToken))[0]
.StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange);
}
/// <summary>U9: a STRING write is NUL-padded at the PLC width (no stale tail) and an over-long one is refused.</summary>
[TwinCATFact]
public async Task String_write_round_trips_and_refuses_overflow()
{
if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason);
await using var drv = NewDriver("tc3-gate-u9-string", native: false,
new TwinCATTagDefinition("S", Host, "GVL_Primitives.vString", TwinCATDataType.String));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var original = (string)(await drv.ReadAsync(["S"], TestContext.Current.CancellationToken))[0].Value!;
try
{
(await drv.WriteAsync([new WriteRequest("S", "Grüße 40°")], TestContext.Current.CancellationToken))[0]
.StatusCode.ShouldBe(0u);
(await drv.ReadAsync(["S"], TestContext.Current.CancellationToken))[0].Value.ShouldBe("Grüße 40°");
(await drv.WriteAsync([new WriteRequest("S", new string('x', 81))], TestContext.Current.CancellationToken))[0]
.StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange, "vString is STRING(80)");
}
finally
{
await drv.WriteAsync([new WriteRequest("S", original)], CancellationToken.None);
}
(await drv.ReadAsync(["S"], TestContext.Current.CancellationToken))[0].Value.ShouldBe(original);
}
}
@@ -1,3 +1,4 @@
using System.Collections.Concurrent;
using System.Runtime.CompilerServices;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
@@ -56,10 +57,22 @@ internal class FakeTwinCATClient : ITwinCATClient
{
ConnectCount++;
if (ThrowOnConnect) throw Exception ?? new InvalidOperationException();
if (BlockConnect is { } block)
{
// Models the real client's blocking SDK Connect: ignores the token until released.
return block.Task.ContinueWith(_ => { IsConnected = true; }, TaskScheduler.Default);
}
IsConnected = true;
return Task.CompletedTask;
}
/// <summary>When set, <see cref="ConnectAsync"/> does not complete (and ignores cancellation) until this
/// is completed — a probe wedged in the synchronous ADS <c>Connect</c>.</summary>
public TaskCompletionSource? BlockConnect { get; set; }
/// <summary>Gets or sets the simulated notification-queue drop count.</summary>
public long DroppedNotificationCount { get; set; }
/// <summary>Simulates reading a value from a symbol.</summary>
/// <param name="symbolPath">The path to the symbol to read.</param>
/// <param name="type">The data type of the symbol.</param>
@@ -70,16 +83,31 @@ internal class FakeTwinCATClient : ITwinCATClient
public virtual Task<(object? value, uint status)> ReadValueAsync(
string symbolPath, TwinCATDataType type, int? bitIndex, int? arrayCount, CancellationToken ct)
{
if (BlockRead is { } block)
{
// A read wedged in the (uncancellable) wire call; completes with the normal answer once released.
return block.Task.ContinueWith(_ =>
{
BlockRead = null;
return ReadValueAsync(symbolPath, type, bitIndex, arrayCount, CancellationToken.None).Result;
}, TaskScheduler.Default);
}
if (ThrowOnRead) throw Exception ?? new InvalidOperationException();
SingleReadCount++;
Interlocked.Increment(ref _singleReadCount);
LastReadArrayCount = arrayCount;
var status = ReadStatuses.TryGetValue(symbolPath, out var s) ? s : TwinCATStatusMapper.Good;
var value = Values.TryGetValue(symbolPath, out var v) ? v : null;
return Task.FromResult((value, status));
}
/// <summary>When set, <see cref="ReadValueAsync"/> does not complete (and ignores cancellation) until this is
/// completed — a poll read wedged in the wire call.</summary>
public TaskCompletionSource? BlockRead { get; set; }
private int _singleReadCount;
/// <summary>Gets the number of individual <see cref="ReadValueAsync"/> calls served.</summary>
public int SingleReadCount { get; private set; }
public int SingleReadCount { get => Volatile.Read(ref _singleReadCount); private set => Volatile.Write(ref _singleReadCount, value); }
/// <summary>Gets the size of each <see cref="ReadValuesAsync"/> batch the driver issued, in order (#645).</summary>
public List<int> SumReadBatches { get; } = new();
/// <summary>Gets or sets an exception <see cref="ReadValuesAsync"/> throws instead of answering — a
@@ -130,21 +158,102 @@ internal class FakeTwinCATClient : ITwinCATClient
string symbolPath, TwinCATDataType type, int? bitIndex, object? value, CancellationToken ct)
{
if (ThrowOnWrite) throw Exception ?? new InvalidOperationException();
if (type is TwinCATDataType.String or TwinCATDataType.WString
&& SymbolShapes.TryGetValue(symbolPath, out var shape))
{
// The real client's encode plan (resolve → CheckRead → codec at the PLC width).
var (planStatus, bytes) = AdsTwinCATClient.PlanStringWrite(type, shape, value, utf8: false);
if (planStatus != TwinCATStatusMapper.Good) return Task.FromResult(planStatus);
RawWrites[symbolPath] = bytes;
}
WriteLog.Add((symbolPath, type, bitIndex, value));
Values[symbolPath] = value;
var status = WriteStatuses.TryGetValue(symbolPath, out var s) ? s : TwinCATStatusMapper.Good;
return Task.FromResult(status);
}
/// <summary>Gets the log of every <see cref="WriteArrayAsync"/> call.</summary>
public List<(string symbol, TwinCATDataType type, IReadOnlyList<object> elements)> ArrayWriteLog { get; } = new();
/// <summary>Simulates an array write: records it, stores the elements and answers from <see cref="WriteStatuses"/>.</summary>
/// <param name="symbolPath">The array symbol path.</param>
/// <param name="type">The declared element type.</param>
/// <param name="elements">The converted elements.</param>
/// <param name="ct">The cancellation token.</param>
/// <returns>The write status.</returns>
public virtual Task<uint> WriteArrayAsync(string symbolPath, TwinCATDataType type, IReadOnlyList<object> elements,
CancellationToken ct)
{
if (ThrowOnWrite) throw Exception ?? new InvalidOperationException();
if (ArrayShapes.TryGetValue(symbolPath, out var array) && SymbolShapes.TryGetValue(symbolPath, out var shape))
{
// The real client's encode plan (CheckArray layout → TwinCATArrayCodec.Encode).
var (planStatus, bytes) = AdsTwinCATClient.PlanArrayWrite(type, elements, shape, array, utf8: false);
if (planStatus != TwinCATStatusMapper.Good) return Task.FromResult(planStatus);
RawWrites[symbolPath] = bytes;
}
ArrayWriteLog.Add((symbolPath, type, elements));
Values[symbolPath] = elements;
return Task.FromResult(WriteStatuses.TryGetValue(symbolPath, out var s) ? s : TwinCATStatusMapper.Good);
}
/// <summary>Resolved PLC symbol shapes; a string / array write to a symbol listed here runs the real client's
/// encode plan and lands its bytes in <see cref="RawWrites"/> (#739 fix round 1).</summary>
public Dictionary<string, TwinCATPlcSymbolShape> SymbolShapes { get; } = new(StringComparer.OrdinalIgnoreCase);
/// <summary>Resolved PLC array shapes for <see cref="SymbolShapes"/> entries that are arrays.</summary>
public Dictionary<string, TwinCATPlcArrayShape> ArrayShapes { get; } = new(StringComparer.OrdinalIgnoreCase);
/// <summary>The raw bytes the encode plan produced, by symbol.</summary>
public ConcurrentDictionary<string, byte[]> RawWrites { get; } = new(StringComparer.OrdinalIgnoreCase);
/// <summary>Raw bytes of bit-write parent words, by parent path (#739 H18). The byte count is the parent's width.</summary>
public Dictionary<string, byte[]> ParentWords { get; } = new(StringComparer.OrdinalIgnoreCase);
/// <summary>Optional parent shapes; absent = an unsigned integer word as wide as its <see cref="ParentWords"/> entry.</summary>
public Dictionary<string, TwinCATPlcSymbolShape> ParentShapes { get; } = new(StringComparer.OrdinalIgnoreCase);
/// <summary>Gets the log of every <see cref="WriteBitAsync"/> call.</summary>
public List<(string parent, int bit, bool value)> BitWriteLog { get; } = new();
/// <summary>When set, <see cref="WriteBitAsync"/> yields between its read and its write so a missing
/// per-parent lock in the driver loses updates.</summary>
public bool YieldInBitWrite { get; set; }
/// <summary>Simulates the parent-width bit RMW with the driver's own <see cref="TwinCATBitAccess.ApplyBitWrite"/>.</summary>
/// <param name="parentPath">The parent word's symbol path.</param>
/// <param name="bit">The bit index.</param>
/// <param name="value">The bit value.</param>
/// <param name="ct">The cancellation token.</param>
/// <returns>The write status.</returns>
public virtual async Task<uint> WriteBitAsync(string parentPath, int bit, bool value, CancellationToken ct)
{
if (ThrowOnWrite) throw Exception ?? new InvalidOperationException();
lock (BitWriteLog) BitWriteLog.Add((parentPath, bit, value));
if (ReadStatuses.TryGetValue(parentPath, out var readStatus) && readStatus != TwinCATStatusMapper.Good)
return readStatus;
byte[] current;
lock (ParentWords) current = ParentWords.TryGetValue(parentPath, out var w) ? [.. w] : [];
var shape = ParentShapes.TryGetValue(parentPath, out var sh)
? sh
: new TwinCATPlcSymbolShape(TwinCATPlcKind.Scalar, current.Length == 0 ? 4 : current.Length, false, "UDINT");
if (YieldInBitWrite) await Task.Yield();
var status = TwinCATBitAccess.ApplyBitWrite(shape, current, bit, value, out var updated);
if (status != TwinCATStatusMapper.Good) return status;
lock (ParentWords) ParentWords[parentPath] = updated;
return WriteStatuses.TryGetValue(parentPath, out var s) ? s : TwinCATStatusMapper.Good;
}
/// <summary>Simulates probing the connection status.</summary>
/// <param name="ct">The cancellation token.</param>
/// <returns>A task that returns the probe result.</returns>
public virtual Task<bool> ProbeAsync(CancellationToken ct)
{
Interlocked.Increment(ref _probeCount);
if (ThrowOnProbe) return Task.FromResult(false);
return Task.FromResult(ProbeResult);
}
private int _probeCount;
/// <summary>Gets how many times <see cref="ProbeAsync"/> ran — a measured window for absence assertions.</summary>
public int ProbeCount => Volatile.Read(ref _probeCount);
/// <summary>Releases unmanaged resources.</summary>
public virtual void Dispose()
{
@@ -160,9 +269,19 @@ internal class FakeTwinCATClient : ITwinCATClient
public bool ThrowOnAddNotification { get; set; }
/// <summary>Per-symbol exceptions <see cref="AddNotificationAsync"/> throws (e.g. a
/// <see cref="TwinCATTypeMismatchException"/> for one refused tag).</summary>
public Dictionary<string, Exception> AddNotificationErrors { get; } = new(StringComparer.OrdinalIgnoreCase);
public ConcurrentDictionary<string, Exception> AddNotificationErrors { get; } = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, int> _addAttempts = new(StringComparer.OrdinalIgnoreCase);
/// <summary>How many times <see cref="AddNotificationAsync"/> was called for <paramref name="symbolPath"/>.</summary>
/// <param name="symbolPath">The symbol path.</param>
/// <returns>The attempt count.</returns>
public int AttemptsFor(string symbolPath) => _addAttempts.TryGetValue(symbolPath, out var n) ? n : 0;
/// <summary>Records the most recently-supplied <c>maxDelayMs</c> for Driver.TwinCAT-014 tests.</summary>
public int LastMaxDelayMs { get; private set; }
/// <summary>When set, <see cref="AddNotificationAsync"/> awaits this gate BEFORE applying
/// <see cref="AddNotificationErrors"/> — wedges an attempt in flight whose outcome is a failure.</summary>
public TaskCompletionSource? PreAttemptGate { get; set; }
/// <summary>When set, <see cref="AddNotificationAsync"/> awaits this gate before creating the handle —
/// lets a test wedge a replay/register mid-flight (STAB-17 orphan race).</summary>
public TaskCompletionSource? AddNotificationGate { get; set; }
@@ -181,6 +300,8 @@ internal class FakeTwinCATClient : ITwinCATClient
string symbolPath, TwinCATDataType type, int? bitIndex, int? arrayCount, TimeSpan cycleTime,
int maxDelayMs, Action<string, object?> onChange, CancellationToken cancellationToken)
{
_addAttempts.AddOrUpdate(symbolPath, 1, (_, n) => n + 1);
if (PreAttemptGate is { } pre) await pre.Task.ConfigureAwait(false);
if (ThrowOnAddNotification)
throw Exception ?? new InvalidOperationException("fake AddNotification failure");
if (AddNotificationErrors.TryGetValue(symbolPath, out var perSymbol))
@@ -0,0 +1,199 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>
/// #739 U9: array writes and STRING writes go through the same codecs the read side decodes with.
/// Arrays were advertised writable but every write threw InvalidCastException (reported BadTypeMismatch);
/// strings bypassed the codec (host-ANSI encoding, no length check against STRING(n)).
/// </summary>
[Trait("Category", "Unit")]
public sealed class TwinCATArrayStringWriteTests
{
private const string Host = "ads://5.23.91.23.1.1:851";
// ---- string codec: encode ----
/// <summary>A STRING(n) encodes NUL-terminated and NUL-padded to the PLC width, in Windows-1252.</summary>
[Fact]
public void EncodeString_is_NUL_padded_Windows1252_at_the_PLC_width()
{
TwinCATStringCodec.EncodeString("AB", 4, utf8: false).ShouldBe(new byte[] { 0x41, 0x42, 0, 0 });
TwinCATStringCodec.EncodeString("ä€", 3, utf8: false).ShouldBe(new byte[] { 0xE4, 0x80, 0 });
TwinCATStringCodec.EncodeString("ä", 3, utf8: true).ShouldBe(new byte[] { 0xC3, 0xA4, 0 });
TwinCATStringCodec.EncodeString("", 2, utf8: false).ShouldBe(new byte[] { 0, 0 });
}
/// <summary>The encode is the decode's inverse (a STRING(n) of exactly n characters fits).</summary>
[Fact]
public void EncodeString_round_trips_through_DecodeString()
{
foreach (var utf8 in new[] { false, true })
{
var bytes = TwinCATStringCodec.EncodeString("Grüße 40°", 81, utf8);
TwinCATStringCodec.DecodeString(bytes, utf8).ShouldBe("Grüße 40°");
}
TwinCATStringCodec.EncodeString("ABC", 4, utf8: false).Length.ShouldBe(4);
}
/// <summary>A value longer than STRING(n), or a character the target encoding cannot hold, is out of range.</summary>
[Fact]
public void EncodeString_refuses_a_value_that_does_not_fit()
{
Should.Throw<OverflowException>(() => TwinCATStringCodec.EncodeString("ABC", 3, utf8: false));
Should.Throw<OverflowException>(() => TwinCATStringCodec.EncodeString("中", 10, utf8: false));
Should.Throw<OverflowException>(() => TwinCATStringCodec.EncodeString("ää", 4, utf8: true)); // 4 bytes + NUL
}
/// <summary>A WSTRING(n) encodes UTF-16LE with a 2-byte terminator and round-trips.</summary>
[Fact]
public void EncodeWString_is_UTF16LE_with_a_terminator()
{
TwinCATStringCodec.EncodeWString("Hi", 6).ShouldBe(new byte[] { 0x48, 0, 0x69, 0, 0, 0 });
TwinCATStringCodec.DecodeWString(TwinCATStringCodec.EncodeWString("中文", 22)).ShouldBe("中文");
Should.Throw<OverflowException>(() => TwinCATStringCodec.EncodeWString("Hi", 4));
}
// ---- array codec: elements + encode ----
/// <summary>
/// A CLR list converts element-wise to the declared type (ScadaBridge writes an <c>int[]</c> to an
/// Int16 node); the element type is converted, not exact-matched.
/// </summary>
[Fact]
public void ToWriteElements_converts_each_element_to_the_declared_type()
{
var elements = TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, new[] { 1, -2, 300 }, 3);
elements.ShouldBe(new object[] { (short)1, (short)-2, (short)300 });
TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Real, new List<object> { 1, 2.5 }, 2)
.ShouldBe(new object[] { 1f, 2.5f });
TwinCATArrayCodec.ToWriteElements(TwinCATDataType.String, new[] { "a", "b" }, 2)
.ShouldBe(new object[] { "a", "b" });
}
/// <summary>A length mismatch and an out-of-range element are out of range; a scalar is a type mismatch.</summary>
[Fact]
public void ToWriteElements_refuses_a_wrong_shape()
{
Should.Throw<OverflowException>(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, new[] { 1, 2 }, 3));
Should.Throw<OverflowException>(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, new[] { 70_000 }, 1));
Should.Throw<InvalidCastException>(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, 5, 1));
Should.Throw<InvalidCastException>(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.String, "abc", 3));
Should.Throw<InvalidCastException>(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, null, 1));
}
/// <summary>Encode lays elements out little-endian at the PLC stride — the exact inverse of Decode.</summary>
[Fact]
public void Encode_is_the_inverse_of_Decode()
{
var intLayout = new TwinCATArrayLayout(2, 2);
var intBytes = TwinCATArrayCodec.Encode(TwinCATDataType.Int, [(short)1, (short)-2], intLayout, utf8: false);
intBytes.ShouldBe(new byte[] { 0x01, 0x00, 0xFE, 0xFF });
RoundTrip(TwinCATDataType.Bool, [true, false, true], new TwinCATArrayLayout(3, 1));
RoundTrip(TwinCATDataType.DInt, [int.MinValue, 7], new TwinCATArrayLayout(2, 4));
RoundTrip(TwinCATDataType.ULInt, [ulong.MaxValue], new TwinCATArrayLayout(1, 8));
RoundTrip(TwinCATDataType.LReal, [1.5, -2.25], new TwinCATArrayLayout(2, 8));
RoundTrip(TwinCATDataType.String, ["ab", "Grüße"], new TwinCATArrayLayout(2, 11));
RoundTrip(TwinCATDataType.WString, ["中", "x"], new TwinCATArrayLayout(2, 8));
// TIME elements arrive as the encoded uint ms (ConvertForWrite) and decode back to double ms.
var time = TwinCATArrayCodec.Encode(TwinCATDataType.Time,
TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Time, new[] { 1500.0, 20.0 }, 2), new TwinCATArrayLayout(2, 4), false);
TwinCATArrayCodec.Decode(TwinCATDataType.Time, time, new TwinCATArrayLayout(2, 4), false)
.ShouldBe(new[] { 1500.0, 20.0 });
}
/// <summary>A string element longer than the element's STRING(n) is out of range.</summary>
[Fact]
public void Encode_refuses_a_string_element_wider_than_its_stride()
{
Should.Throw<OverflowException>(() =>
TwinCATArrayCodec.Encode(TwinCATDataType.String, ["toolong"], new TwinCATArrayLayout(1, 4), utf8: false));
Should.Throw<ArgumentException>(() =>
TwinCATArrayCodec.Encode(TwinCATDataType.Int, [(short)1], new TwinCATArrayLayout(2, 2), utf8: false));
}
private static void RoundTrip(TwinCATDataType type, object[] elements, TwinCATArrayLayout layout)
{
var bytes = TwinCATArrayCodec.Encode(type, elements, layout, utf8: false);
bytes.Length.ShouldBe(layout.ByteLength);
var decoded = TwinCATArrayCodec.Decode(type, bytes, layout, utf8: false);
decoded.Cast<object>().ShouldBe(elements);
}
// ---- driver ----
private static (TwinCATDriver drv, FakeTwinCATClient fake) NewDriver(params TwinCATTagDefinition[] tags)
{
var fake = new FakeTwinCATClient();
var drv = new TwinCATDriver(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host)],
RawTags = TwinCATRawTags.Entries(tags),
Probe = new TwinCATProbeOptions { Enabled = false },
}, "drv-1", new FakeTwinCATClientFactory { Customise = () => fake });
return (drv, fake);
}
/// <summary>An array tag's write sends ArrayLength converted elements through WriteArrayAsync.</summary>
[Fact]
public async Task An_array_write_sends_converted_elements()
{
var (drv, fake) = NewDriver(new TwinCATTagDefinition("Buf", Host, "MAIN.Buf", TwinCATDataType.Int, ArrayLength: 3));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var result = await drv.WriteAsync([new WriteRequest("Buf", new[] { 1, 2, 3 })], TestContext.Current.CancellationToken);
result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.Good);
var write = fake.ArrayWriteLog.ShouldHaveSingleItem();
write.symbol.ShouldBe("MAIN.Buf");
write.elements.ShouldBe(new object[] { (short)1, (short)2, (short)3 });
fake.WriteLog.ShouldBeEmpty(); // never the scalar path (that threw InvalidCastException)
}
/// <summary>
/// A length mismatch is BadOutOfRange and a scalar value BadTypeMismatch — nothing reaches the wire
/// and health is untouched (a refusal is not a comms fault). Positive control: the good write above.
/// </summary>
[Fact]
public async Task A_wrong_shape_array_write_is_refused_before_the_wire()
{
var (drv, fake) = NewDriver(new TwinCATTagDefinition("Buf", Host, "MAIN.Buf", TwinCATDataType.Int, ArrayLength: 3));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var results = await drv.WriteAsync(
[
new WriteRequest("Buf", new[] { 1, 2 }),
new WriteRequest("Buf", 5),
new WriteRequest("Buf", new[] { 1, 2, 70_000 }),
], TestContext.Current.CancellationToken);
results.Select(r => r.StatusCode).ShouldBe(
[
TwinCATStatusMapper.BadOutOfRange,
TwinCATStatusMapper.BadTypeMismatch,
TwinCATStatusMapper.BadOutOfRange,
]);
fake.ArrayWriteLog.ShouldBeEmpty();
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
/// <summary>An over-long string surfaced by the client as OverflowException is BadOutOfRange, not a comms fault.</summary>
[Fact]
public async Task An_over_long_string_write_is_BadOutOfRange()
{
var (drv, fake) = NewDriver(new TwinCATTagDefinition("Name", Host, "MAIN.Name", TwinCATDataType.String));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
fake.ThrowOnWrite = true;
fake.Exception = new OverflowException("6 bytes do not fit STRING(4)");
var result = await drv.WriteAsync([new WriteRequest("Name", "toolong")], TestContext.Current.CancellationToken);
result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
}
@@ -0,0 +1,89 @@
using System.Collections.Concurrent;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>
/// #739 / R02:P1 — TwinCAT is an <see cref="IBatchSubscribable"/>. DriverInstanceActor subscribes ONLY the
/// batch event of a batch-capable driver, so every data change — poll cycle AND native push, including the
/// Bad snapshots the native path publishes itself — must reach <c>OnDataChangeBatch</c>; the per-tag event
/// keeps firing for legacy consumers (the CLI).
/// </summary>
[Trait("Category", "Unit")]
public sealed class TwinCATBatchSubscribeTests
{
private const string Host = "ads://5.23.91.23.1.1:851";
private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, bool native) =>
new(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host)],
RawTags = TwinCATRawTags.Entries(
[
new TwinCATTagDefinition("A", Host, "MAIN.A", TwinCATDataType.DInt),
new TwinCATTagDefinition("B", Host, "MAIN.B", TwinCATDataType.DInt),
]),
Probe = new TwinCATProbeOptions { Enabled = false },
UseNativeNotifications = native,
}, "drv-1", factory);
/// <summary>The driver advertises the batch capability.</summary>
[Fact]
public void Driver_is_batch_subscribable() =>
NewDriver(new FakeTwinCATClientFactory(), native: false).ShouldBeAssignableTo<IBatchSubscribable>();
/// <summary>A poll cycle that changes two tags raises ONE batch carrying both, and the per-tag event twice.</summary>
[Fact]
public async Task A_poll_cycle_raises_one_batch_with_every_changed_tag()
{
var factory = new FakeTwinCATClientFactory
{
Customise = () => new FakeTwinCATClient { Values = { ["MAIN.A"] = 1, ["MAIN.B"] = 2 } },
};
var drv = NewDriver(factory, native: false);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var batches = new ConcurrentQueue<DataChangeBatchEventArgs>();
var single = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChangeBatch += (_, e) => batches.Enqueue(e);
drv.OnDataChange += (_, e) => single.Enqueue(e);
var handle = await drv.SubscribeAsync(["A", "B"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
await WaitForAsync(() => !batches.IsEmpty, TimeSpan.FromSeconds(3));
await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken);
var first = batches.First();
first.SubscriptionHandle.ShouldBe(handle);
first.Items.Select(i => i.FullReference).ShouldBe(["A", "B"], ignoreOrder: true);
single.Count(e => e.FullReference is "A" or "B").ShouldBeGreaterThanOrEqualTo(2);
}
/// <summary>A native push and the native path's own Bad snapshot both reach the batch event.</summary>
[Fact]
public async Task Native_pushes_and_native_bad_snapshots_reach_the_batch_event()
{
var fake = new FakeTwinCATClient();
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, native: true);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var batches = new ConcurrentQueue<DataChangeBatchEventArgs>();
drv.OnDataChangeBatch += (_, e) => batches.Enqueue(e);
_ = await drv.SubscribeAsync(["A", "Ghost"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
fake.FireNotification("MAIN.A", 42);
var items = batches.SelectMany(b => b.Items).ToList();
items.Single(i => i.FullReference == "Ghost").Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
var a = items.Single(i => i.FullReference == "A");
a.Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good);
a.Snapshot.Value.ShouldBe(42);
}
private static async Task WaitForAsync(Func<bool> condition, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (!condition() && DateTime.UtcNow < deadline)
await Task.Delay(20);
}
}
@@ -55,4 +55,50 @@ public sealed class TwinCATBitAccessTests
TwinCATBitAccess.CheckParent(0, TwinCATPlcSymbolShape.Classify("ST_X", DataTypeCategory.Struct, 8)).ShouldNotBeNull();
TwinCATBitAccess.CheckParent(31, TwinCATPlcSymbolShape.Classify("DWORD", DataTypeCategory.Primitive, 4)).ShouldBeNull();
}
// ---- #739 H18: the bit WRITE is a read-modify-write at the parent's own width ----
/// <summary>A bit write flips exactly one bit of the parent's raw bytes at the parent's width.</summary>
[Fact]
public void ApplyBitWrite_flips_one_bit_at_the_parent_width()
{
var word = TwinCATPlcSymbolShape.Classify("WORD", DataTypeCategory.Primitive, 2);
TwinCATBitAccess.ApplyBitWrite(word, [0x01, 0x00], 9, true, out var set).ShouldBe(TwinCATStatusMapper.Good);
set.ShouldBe(new byte[] { 0x01, 0x02 });
TwinCATBitAccess.ApplyBitWrite(word, [0xFF, 0xFF], 3, false, out var cleared).ShouldBe(TwinCATStatusMapper.Good);
cleared.ShouldBe(new byte[] { 0xF7, 0xFF });
}
/// <summary>
/// A signed parent with its sign bit set is plain bytes — the old UDInt round-trip read it back as
/// a value ≥ 2³¹ and a checked conversion turned ANY bit write into BadOutOfRange.
/// </summary>
[Fact]
public void ApplyBitWrite_on_a_negative_signed_parent_preserves_the_sign_bit()
{
var intWord = TwinCATPlcSymbolShape.Classify("INT", DataTypeCategory.Primitive, 2);
TwinCATBitAccess.ApplyBitWrite(intWord, [0x00, 0x80], 0, true, out var updated).ShouldBe(TwinCATStatusMapper.Good);
updated.ShouldBe(new byte[] { 0x01, 0x80 });
}
/// <summary>A bit beyond the parent is BadOutOfRange; a non-integer parent BadTypeMismatch; nothing is produced.</summary>
[Fact]
public void ApplyBitWrite_refuses_a_bit_the_parent_does_not_have()
{
var word = TwinCATPlcSymbolShape.Classify("UINT", DataTypeCategory.Primitive, 2);
TwinCATBitAccess.ApplyBitWrite(word, [0, 0], 16, true, out var none).ShouldBe(TwinCATStatusMapper.BadOutOfRange);
none.ShouldBeEmpty();
var real = TwinCATPlcSymbolShape.Classify("REAL", DataTypeCategory.Primitive, 4);
TwinCATBitAccess.ApplyBitWrite(real, [0, 0, 0, 0], 1, true, out _).ShouldBe(TwinCATStatusMapper.BadTypeMismatch);
}
/// <summary>A parent read shorter than the parent's width is never written back (it would zero bytes).</summary>
[Fact]
public void ApplyBitWrite_refuses_a_short_parent_buffer()
{
var dword = TwinCATPlcSymbolShape.Classify("DWORD", DataTypeCategory.Primitive, 4);
TwinCATBitAccess.ApplyBitWrite(dword, [0xFF, 0xFF], 1, true, out var none)
.ShouldBe(TwinCATStatusMapper.BadCommunicationError);
none.ShouldBeEmpty();
}
}
@@ -0,0 +1,26 @@
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>Collects the driver's Warning-level messages (thread-safe) for once-per-X assertions.</summary>
internal sealed class TwinCATCapturingLogger : ILogger<TwinCATDriver>
{
private readonly List<string> _warnings = [];
/// <summary>Gets a snapshot of the formatted Warning messages logged so far.</summary>
public IReadOnlyList<string> Warnings { get { lock (_warnings) return [.. _warnings]; } }
/// <inheritdoc />
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
/// <inheritdoc />
public bool IsEnabled(LogLevel logLevel) => true;
/// <inheritdoc />
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
Func<TState, Exception?, string> formatter)
{
if (logLevel == LogLevel.Warning) lock (_warnings) _warnings.Add(formatter(state, exception));
}
}
@@ -0,0 +1,163 @@
using System.Collections.Concurrent;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>
/// The driver health contract (#739 H12): data proving the device answered — a native notification or
/// a successful probe — advances <c>LastSuccessfulRead</c>; sustained probe/connect failure degrades with
/// it preserved; a refusal (a write the client cannot express, a replay the PLC refuses) is no comms fault.
/// </summary>
[Trait("Category", "Unit")]
public sealed class TwinCATHealthContractTests
{
private const string Host = "ads://5.23.91.23.1.1:851";
private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, TwinCATProbeOptions probe, bool native = true) =>
new(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host)],
RawTags = TwinCATRawTags.Entries([new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt)]),
Probe = probe,
UseNativeNotifications = native,
}, "drv-1", factory);
private static TwinCATProbeOptions FastProbe => new()
{
Enabled = true,
Interval = TimeSpan.FromMilliseconds(30),
Timeout = TimeSpan.FromMilliseconds(100),
};
/// <summary>A native push advances <c>LastSuccessfulRead</c> and restores Healthy after a comms-fault write.</summary>
[Fact]
public async Task A_native_notification_advances_LastSuccessfulRead_and_restores_Healthy()
{
var fake = new FakeTwinCATClient();
var factory = new FakeTwinCATClientFactory { Customise = () => fake };
var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false });
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead;
initStamp.ShouldNotBeNull();
// A write that fails on the wire degrades (comms fault) with the stamp preserved.
fake.ThrowOnWrite = true;
_ = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
drv.GetHealth().LastSuccessfulRead.ShouldBe(initStamp);
// Condition, not a fixed delay: the clock only has to move past the stamp (the Degraded state means
// the notification is not inside the Healthy throttle window).
await WaitForAsync(() => DateTime.UtcNow > initStamp!.Value, TimeSpan.FromSeconds(2));
fake.FireNotification("MAIN.Speed", 7);
var after = drv.GetHealth();
after.State.ShouldBe(DriverState.Healthy);
after.LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp!.Value);
}
/// <summary>A successful probe advances <c>LastSuccessfulRead</c> in a driver that serves only notifications.</summary>
[Fact]
public async Task A_successful_probe_advances_LastSuccessfulRead()
{
var factory = new FakeTwinCATClientFactory();
var drv = NewDriver(factory, FastProbe);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead!.Value;
await WaitForAsync(() => drv.GetHealth().LastSuccessfulRead > initStamp.AddMilliseconds(250), TimeSpan.FromSeconds(3));
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp.AddMilliseconds(250));
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
/// <summary>
/// An unreachable AMS router (every probe connect throws) degrades the driver after
/// <see cref="TwinCATDriver.ProbeFailuresBeforeDegraded"/> consecutive failures, preserving the stamp —
/// before, the probe caught it and health stayed Healthy forever.
/// </summary>
[Fact]
public async Task Consecutive_probe_failures_degrade_preserving_LastSuccessfulRead()
{
var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { ThrowOnConnect = true } };
var drv = NewDriver(factory, FastProbe);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead;
await WaitForAsync(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(3));
var health = drv.GetHealth();
health.State.ShouldBe(DriverState.Degraded);
health.LastSuccessfulRead.ShouldBe(initStamp);
factory.Clients.Count.ShouldBeGreaterThanOrEqualTo(TwinCATDriver.ProbeFailuresBeforeDegraded);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
/// <summary>
/// A write the client refuses with <see cref="NotSupportedException"/> is <c>BadNotSupported</c> with no
/// health write (#611: a refusal is not a comms fault). Positive control: a wire failure does degrade.
/// </summary>
[Fact]
public async Task A_NotSupported_write_leaves_health_alone()
{
var fake = new FakeTwinCATClient { ThrowOnWrite = true, Exception = new NotSupportedException("no") };
var factory = new FakeTwinCATClientFactory { Customise = () => fake };
var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false }, native: false);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var result = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken);
result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadNotSupported);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
fake.Exception = new InvalidOperationException("wire");
result = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken);
result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
}
/// <summary>
/// A reconnect replay the PLC REFUSES (an isolated type-mismatch tag re-attempted on every reconnect)
/// republishes the refusal's status and does not degrade. Positive control: a replay that fails on the
/// wire degrades (covered for the same flow in <c>TwinCATReconnectReplayTests</c>).
/// </summary>
[Fact]
public async Task A_refused_replay_republishes_the_refusal_status_without_degrading()
{
var build = 0;
var factory = new FakeTwinCATClientFactory
{
Customise = () =>
{
var c = new FakeTwinCATClient();
if (++build >= 2)
c.AddNotificationErrors["MAIN.Speed"] = new TwinCATTypeMismatchException("MAIN.Speed", "now a STRING");
return c;
},
};
var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false });
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
_ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
factory.Clients[0].SimulateWireDrop();
// A write reconnects (and replays) without itself touching health on success.
_ = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken);
factory.Clients.Count.ShouldBe(2);
events.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadTypeMismatch);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
private static async Task WaitForAsync(Func<bool> condition, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (!condition() && DateTime.UtcNow < deadline)
await Task.Delay(20);
}
}
@@ -0,0 +1,298 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>
/// #739 L8 + the Lane A runtime contract (final stop = <c>ShutdownAsync</c> then <c>DisposeAsync</c>,
/// bounded 5 s + 250 ms): a probe wedged in the blocking ADS connect cannot hang teardown; dispose may run
/// while a shutdown is still running; an abandoned initialise overlapping the next shutdown never installs
/// its generation; a value queued before unsubscribe is not delivered after it; re-init swaps its tables
/// and retracts the previous generation's notifications; queue drops are reported.
/// </summary>
[Trait("Category", "Unit")]
public sealed class TwinCATLifecycleTests
{
private const string Host = "ads://5.23.91.23.1.1:851";
private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, bool probe = true, bool native = true,
TwinCATCapturingLogger? log = null, params TwinCATTagDefinition[] tags) =>
new(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host)],
RawTags = TwinCATRawTags.Entries(tags.Length > 0
? tags
: [new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt)]),
Probe = probe
? new TwinCATProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(30), Timeout = TimeSpan.FromMilliseconds(100) }
: new TwinCATProbeOptions { Enabled = false },
UseNativeNotifications = native,
}, "drv-1", factory, log);
private static async Task WaitForAsync(Func<bool> condition, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (!condition() && DateTime.UtcNow < deadline)
await Task.Delay(10);
}
/// <summary>
/// A probe wedged in the (uncancellable) connect no longer hangs ShutdownAsync — the join is bounded —
/// and the client the wedged connect eventually produces is disposed, not installed on a dead device.
/// </summary>
[Fact]
public async Task Shutdown_is_bounded_when_a_probe_is_wedged_in_connect()
{
var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { BlockConnect = wedge } };
var drv = NewDriver(factory);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2));
factory.Clients.Count.ShouldBe(1); // the probe is inside the wedged connect
var sw = Stopwatch.StartNew();
await drv.ShutdownAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ShutdownBudget + TimeSpan.FromMilliseconds(500));
wedge.SetResult();
await WaitForAsync(() => factory.Clients[0].DisposeCount == 1, TimeSpan.FromSeconds(2));
factory.Clients[0].DisposeCount.ShouldBe(1);
drv.DeviceCount.ShouldBe(0);
}
/// <summary>
/// DisposeAsync while a ShutdownAsync is still running (joining a wedged probe) returns promptly and
/// both complete without throwing; the device is released exactly once.
/// </summary>
[Fact]
public async Task Dispose_concurrent_with_a_running_shutdown_completes_promptly()
{
var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { BlockConnect = wedge } };
var drv = NewDriver(factory);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2));
var shutdown = drv.ShutdownAsync(CancellationToken.None);
var sw = Stopwatch.StartNew();
await drv.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
// Ordering is the real assertion: dispose finished while the first shutdown was still joining the
// wedged probe. The elapsed bound is generous (the runtime allows 250 ms; a loaded CI box gets 1 s).
shutdown.IsCompleted.ShouldBeFalse("the positive control: the first shutdown really was still joining the probe");
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1));
wedge.SetResult();
await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
drv.Dispose(); // and a late sync Dispose is a no-op
drv.GetHealth().State.ShouldBe(DriverState.Unknown);
}
/// <summary>
/// An InitializeAsync overtaken by a ShutdownAsync before it commits must not install its devices or
/// start probe loops when it resumes. Positive control: a fresh initialise afterwards installs normally.
/// </summary>
[Fact]
public async Task An_initialise_overtaken_by_shutdown_before_commit_installs_nothing()
{
var factory = new FakeTwinCATClientFactory();
var drv = NewDriver(factory);
var paused = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var resume = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
drv.BeforeCommitForTest = async () => { paused.TrySetResult(); await resume.Task; };
var abandoned = drv.InitializeAsync("{}", CancellationToken.None);
await paused.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
resume.SetResult();
await Should.ThrowAsync<OperationCanceledException>(() => abandoned.WaitAsync(TimeSpan.FromSeconds(5)));
drv.DeviceCount.ShouldBe(0);
factory.Clients.ShouldBeEmpty();
drv.GetHealth().State.ShouldBe(DriverState.Unknown);
// Positive control + measured window: a fresh generation's probe creates exactly one client and
// ticks several times; an overtaken-generation probe loop (same interval, started earlier) would have
// created a second client within that window.
drv.BeforeCommitForTest = null;
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
drv.DeviceCount.ShouldBe(1);
await WaitForAsync(() => factory.Clients.Count >= 1 && factory.Clients[0].ProbeCount >= 4, TimeSpan.FromSeconds(3));
factory.Clients[0].ProbeCount.ShouldBeGreaterThanOrEqualTo(4);
factory.Clients.Count.ShouldBe(1, "no probe loop of the overtaken generation may be running");
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
/// <summary>
/// The Lane A shape: an InitializeAsync the runtime abandoned while it is still releasing the previous
/// generation (joining a probe wedged in connect) overlaps the next ShutdownAsync. Shutdown stays
/// bounded and, once everything settles, no device, probe loop or live client survives.
/// </summary>
[Fact]
public async Task An_abandoned_initialise_overlapping_shutdown_leaves_nothing_running()
{
var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var first = true;
var factory = new FakeTwinCATClientFactory
{
Customise = () =>
{
var c = new FakeTwinCATClient { BlockConnect = first ? wedge : null };
first = false;
return c;
},
};
var drv = NewDriver(factory);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2));
var abandoned = drv.InitializeAsync("{}", CancellationToken.None); // joins the wedged generation-1 probe
await WaitForAsync(() => drv.DeviceCount == 1 && factory.Clients.Count >= 2, TimeSpan.FromSeconds(3)); // gen 2 committed + probing
var sw = Stopwatch.StartNew();
await drv.ShutdownAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ShutdownBudget + TimeSpan.FromMilliseconds(500));
wedge.SetResult();
try { await abandoned.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); }
catch (OperationCanceledException) when (!TestContext.Current.CancellationToken.IsCancellationRequested) { }
drv.DeviceCount.ShouldBe(0);
await WaitForAsync(() => factory.Clients.All(c => c.DisposeCount > 0 || !c.IsConnected), TimeSpan.FromSeconds(2));
var settled = factory.Clients.Count;
// Measured window: a sibling driver on the same probe interval ticks 4 times; any surviving probe
// loop of the torn-down driver would have built a new client in that time.
var witnessFactory = new FakeTwinCATClientFactory();
var witness = NewDriver(witnessFactory);
await witness.InitializeAsync("{}", TestContext.Current.CancellationToken);
await WaitForAsync(() => witnessFactory.Clients.Count >= 1 && witnessFactory.Clients[0].ProbeCount >= 4, TimeSpan.FromSeconds(3));
witnessFactory.Clients[0].ProbeCount.ShouldBeGreaterThanOrEqualTo(4);
await witness.ShutdownAsync(TestContext.Current.CancellationToken);
factory.Clients.Count.ShouldBe(settled, "no probe loop may still be creating clients");
factory.Clients.ShouldAllBe(c => c.DisposeCount > 0 || !c.IsConnected);
}
/// <summary>
/// A value the client queued before the unsubscribe is dropped if dispatched after it.
/// Positive control: the same callback delivers while subscribed.
/// </summary>
[Fact]
public async Task A_notification_dispatched_after_unsubscribe_is_dropped()
{
var factory = new FakeTwinCATClientFactory();
var drv = NewDriver(factory, probe: false);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
var handle = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
var callback = factory.Clients[0].Notifications.Single().OnChange; // what the client's queue holds
callback("MAIN.Speed", 1);
events.Count.ShouldBe(1);
await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken);
callback("MAIN.Speed", 2);
events.Count.ShouldBe(1);
}
/// <summary>
/// A direct re-initialise with a changed config swaps the tag table (the removed tag is unknown, the
/// added one resolves) and retracts the previous generation's live notifications.
/// </summary>
[Fact]
public async Task Reinitialise_swaps_tables_and_retracts_the_previous_generations_notifications()
{
var factory = new FakeTwinCATClientFactory();
var drv = NewDriver(factory, probe: false);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
factory.Clients[0].Notifications.Count.ShouldBe(1);
var next = System.Text.Json.JsonSerializer.Serialize(new
{
devices = new[] { new { hostAddress = Host } },
rawTags = new[]
{
new
{
rawPath = "Torque",
tagConfig = TwinCATTagDefinitionFactory.ToTagConfig(
new TwinCATTagDefinition("Torque", Host, "MAIN.Torque", TwinCATDataType.DInt)),
writeIdempotent = false,
deviceName = Host,
},
},
probe = new { enabled = false },
});
await drv.InitializeAsync(next, TestContext.Current.CancellationToken);
factory.Clients[0].Notifications.ShouldBeEmpty();
factory.Clients[0].DisposeCount.ShouldBe(1);
(await drv.ReadAsync(["Speed"], TestContext.Current.CancellationToken)).Single().StatusCode
.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
(await drv.ReadAsync(["Torque"], TestContext.Current.CancellationToken)).Single().StatusCode
.ShouldBe(TwinCATStatusMapper.Good);
}
/// <summary>Reads racing a stream of re-initialises never throw (no table is ever mid-Clear).</summary>
[Fact]
public async Task Reads_racing_reinitialise_never_throw()
{
var factory = new FakeTwinCATClientFactory();
var drv = NewDriver(factory, probe: false);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
// Run for at least 300 ms of overlap AND until the reader has seen a Good read mid-churn (positive
// control), bounded at 10 s — a condition, not a fixed window, so a loaded machine cannot starve it.
var good = 0;
var reads = 0;
var inits = 0;
var minUntil = DateTime.UtcNow.AddMilliseconds(300);
var deadline = DateTime.UtcNow.AddSeconds(10);
bool Done() => (DateTime.UtcNow > minUntil && Volatile.Read(ref good) > 0 && Volatile.Read(ref inits) > 10)
|| DateTime.UtcNow > deadline;
var reader = Task.Run(async () =>
{
while (!Done())
{
var r = await drv.ReadAsync(["Speed"], CancellationToken.None);
Interlocked.Increment(ref reads);
if (r[0].StatusCode == TwinCATStatusMapper.Good) Interlocked.Increment(ref good);
}
}, TestContext.Current.CancellationToken);
while (!Done())
{
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
Interlocked.Increment(ref inits);
await Task.Yield(); // let the reader in between generations
}
await reader; // throws if any read threw
good.ShouldBeGreaterThan(0);
inits.ShouldBeGreaterThan(10);
}
/// <summary>Notification-queue drops are reported once per probe tick that sees new drops.</summary>
[Fact]
public async Task Notification_queue_drops_are_reported()
{
var fake = new FakeTwinCATClient();
var log = new TwinCATCapturingLogger();
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, log: log);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
await WaitForAsync(() => fake.ProbeCount >= 3, TimeSpan.FromSeconds(3));
log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(0); // 3 ticks, no drops, no warning
fake.DroppedNotificationCount = 5;
await WaitForAsync(() => log.Warnings.Any(w => w.Contains("dropped 5")), TimeSpan.FromSeconds(2));
var ticks = fake.ProbeCount;
await WaitForAsync(() => fake.ProbeCount >= ticks + 3, TimeSpan.FromSeconds(3));
log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(1); // 3 more ticks, no new drops, no repeat
fake.DroppedNotificationCount = 8;
await WaitForAsync(() => log.Warnings.Any(w => w.Contains("dropped 3")), TimeSpan.FromSeconds(2));
log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(2);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
}
@@ -0,0 +1,225 @@
using System.Collections.Concurrent;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>
/// Native-subscribe isolation (#739): an unresolvable reference publishes <c>BadNodeIdUnknown</c>
/// (U12) instead of keeping whatever status the server seeded, and one failing registration does not
/// take down the rest of the subscription (H11).
/// </summary>
[Trait("Category", "Unit")]
public sealed class TwinCATNativeIsolationTests
{
private const string Host = "ads://5.23.91.23.1.1:851";
private static TwinCATDriver NewNativeDriver(FakeTwinCATClientFactory factory, params TwinCATTagDefinition[] tags) =>
new(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host)],
RawTags = TwinCATRawTags.Entries(tags),
Probe = new TwinCATProbeOptions { Enabled = false },
UseNativeNotifications = true,
}, "drv-1", factory);
/// <summary>
/// U12: an unknown reference and a tag routed to an unconfigured device each publish one
/// <c>BadNodeIdUnknown</c> snapshot; the resolvable tag still registers and delivers Good (positive control).
/// </summary>
[Fact]
public async Task Native_subscribe_publishes_BadNodeIdUnknown_for_unresolvable_references()
{
var factory = new FakeTwinCATClientFactory();
var drv = NewNativeDriver(factory,
new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt),
new TwinCATTagDefinition("Orphan", "ads://9.9.9.9.1.1:851", "MAIN.Orphan", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
_ = await drv.SubscribeAsync(["Ghost", "Speed", "Orphan"], TimeSpan.FromMilliseconds(100),
TestContext.Current.CancellationToken);
var ghost = events.Single(e => e.FullReference == "Ghost");
ghost.Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
ghost.Snapshot.Value.ShouldBeNull();
events.Single(e => e.FullReference == "Orphan").Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
// Positive control: the resolvable tag registered and its push arrives Good.
factory.Clients.ShouldHaveSingleItem().Notifications.ShouldHaveSingleItem().SymbolPath.ShouldBe("MAIN.Speed");
factory.Clients[0].FireNotification("MAIN.Speed", 42);
var good = events.Single(e => e.FullReference == "Speed");
good.Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good);
good.Snapshot.Value.ShouldBe(42);
}
/// <summary>
/// H11: one tag whose AddDeviceNotification round-trip fails (not a refusal) goes Bad on its own and
/// the subscribe call SUCCEEDS — before, the exception disposed every registration and rethrew, so one
/// bad tag served zero data for the whole driver. The good tags keep delivering (positive control).
/// </summary>
[Fact]
public async Task One_failing_registration_does_not_tear_down_the_subscription()
{
var fake = new FakeTwinCATClient();
fake.AddNotificationErrors["MAIN.B"] = new InvalidOperationException("AddDeviceNotificationEx blew up");
var factory = new FakeTwinCATClientFactory { Customise = () => fake };
var drv = NewNativeDriver(factory,
new TwinCATTagDefinition("A", Host, "MAIN.A", TwinCATDataType.DInt),
new TwinCATTagDefinition("B", Host, "MAIN.B", TwinCATDataType.DInt),
new TwinCATTagDefinition("C", Host, "MAIN.C", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
var handle = await drv.SubscribeAsync(["A", "B", "C"], TimeSpan.FromMilliseconds(100),
TestContext.Current.CancellationToken);
fake.Notifications.Select(n => n.SymbolPath).ShouldBe(["MAIN.A", "MAIN.C"], ignoreOrder: true);
events.ShouldHaveSingleItem().FullReference.ShouldBe("B");
events.Single().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError);
fake.FireNotification("MAIN.C", 3);
events.Last().FullReference.ShouldBe("C");
events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good);
// The failed tag is kept as a dead intent and unsubscribe still removes everything.
drv.GetDeviceState(Host)!.NativeRegistrations.Count.ShouldBe(3);
await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken);
drv.GetDeviceState(Host)!.NativeRegistrations.ShouldBeEmpty();
fake.Notifications.ShouldBeEmpty();
}
/// <summary>
/// H11: a refusal (symbol not found — the device answered) isolates the tag with the refusal's own
/// status and leaves driver health alone; the good tag delivers.
/// </summary>
[Fact]
public async Task A_refused_symbol_is_isolated_without_a_health_write()
{
var fake = new FakeTwinCATClient();
fake.AddNotificationErrors["MAIN.Typo"] = new TwinCATNotificationRefusedException(
"MAIN.Typo", "resolving the symbol failed", TwinCATStatusMapper.BadNodeIdUnknown);
var factory = new FakeTwinCATClientFactory { Customise = () => fake };
var drv = NewNativeDriver(factory,
new TwinCATTagDefinition("Typo", Host, "MAIN.Typo", TwinCATDataType.DInt),
new TwinCATTagDefinition("Good", Host, "MAIN.Good", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
_ = await drv.SubscribeAsync(["Typo", "Good"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
events.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
fake.FireNotification("MAIN.Good", 1);
events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good);
}
/// <summary>
/// H11: in a two-device driver an unreachable device only blacks out ITS tags (Bad + dead intents,
/// health Degraded); the reachable device's tags register and deliver.
/// </summary>
[Fact]
public async Task An_unreachable_device_blacks_out_only_its_own_tags()
{
const string hostB = "ads://5.23.91.24.1.1:851";
var build = 0;
var factory = new FakeTwinCATClientFactory
{
// Device A's client is created first and works; device B's connect throws.
Customise = () => new FakeTwinCATClient { ThrowOnConnect = ++build == 2 },
};
var drv = new TwinCATDriver(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host), new TwinCATDeviceOptions(hostB)],
RawTags = TwinCATRawTags.Entries(
[
new TwinCATTagDefinition("A1", Host, "MAIN.A1", TwinCATDataType.DInt),
new TwinCATTagDefinition("B1", hostB, "MAIN.B1", TwinCATDataType.DInt),
new TwinCATTagDefinition("B2", hostB, "MAIN.B2", TwinCATDataType.DInt),
new TwinCATTagDefinition("A2", Host, "MAIN.A2", TwinCATDataType.DInt),
]),
Probe = new TwinCATProbeOptions { Enabled = false },
UseNativeNotifications = true,
}, "drv-1", factory);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
_ = await drv.SubscribeAsync(["A1", "B1", "B2", "A2"], TimeSpan.FromMilliseconds(100),
TestContext.Current.CancellationToken);
factory.Clients.Count.ShouldBe(2); // B2 did not attempt a second connect
factory.Clients[0].Notifications.Select(n => n.SymbolPath).ShouldBe(["MAIN.A1", "MAIN.A2"], ignoreOrder: true);
events.Select(e => e.FullReference).ShouldBe(["B1", "B2"], ignoreOrder: true);
events.ShouldAllBe(e => e.Snapshot.StatusCode == TwinCATStatusMapper.BadCommunicationError);
drv.GetDeviceState(hostB)!.NativeRegistrations.Count.ShouldBe(2);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
factory.Clients[0].FireNotification("MAIN.A2", 22);
events.Last().FullReference.ShouldBe("A2");
events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good);
}
/// <summary>
/// H11: the dead intent left by a failed registration is re-registered by the probe-tick retry, and
/// its first push then arrives Good — the subscription heals without a re-subscribe.
/// </summary>
[Fact]
public async Task A_failed_registration_is_recovered_by_the_probe_tick_retry()
{
var fake = new FakeTwinCATClient();
fake.AddNotificationErrors["MAIN.B"] = new InvalidOperationException("transient");
var factory = new FakeTwinCATClientFactory { Customise = () => fake };
var drv = new TwinCATDriver(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host)],
RawTags = TwinCATRawTags.Entries([new TwinCATTagDefinition("B", Host, "MAIN.B", TwinCATDataType.DInt)]),
Probe = new TwinCATProbeOptions
{
Enabled = true,
Interval = TimeSpan.FromMilliseconds(40),
Timeout = TimeSpan.FromMilliseconds(100),
},
UseNativeNotifications = true,
}, "drv-1", factory);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
_ = await drv.SubscribeAsync(["B"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
fake.AddNotificationErrors.Clear();
await WaitForAsync(() => fake.Notifications.Count == 1, TimeSpan.FromSeconds(3));
fake.Notifications.Count.ShouldBe(1);
fake.FireNotification("MAIN.B", 5);
events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
/// <summary>A comms-class registration status stays an exception; any other status is a per-tag refusal.</summary>
[Fact]
public void Registration_failures_are_classified_by_status()
{
AdsTwinCATClient.RegistrationFailure("MAIN.X", "step", TwinCATStatusMapper.BadNodeIdUnknown)
.ShouldBeOfType<TwinCATNotificationRefusedException>().StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
AdsTwinCATClient.RegistrationFailure("MAIN.X", "step", TwinCATStatusMapper.BadOutOfService)
.ShouldBeOfType<TwinCATNotificationRefusedException>();
AdsTwinCATClient.RegistrationFailure("MAIN.X", "step", TwinCATStatusMapper.BadCommunicationError)
.ShouldBeOfType<InvalidOperationException>();
AdsTwinCATClient.RegistrationFailure("MAIN.X", "step", TwinCATStatusMapper.BadTimeout)
.ShouldBeOfType<InvalidOperationException>();
}
private static async Task WaitForAsync(Func<bool> condition, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (!condition() && DateTime.UtcNow < deadline)
await Task.Delay(20);
}
}
@@ -98,13 +98,14 @@ public sealed class TwinCATNativeNotificationTests
events.Count.ShouldBe(afterUnsub);
}
/// <summary>Verifies that native subscribe failure mid-registration cleans up partial state.</summary>
/// <summary>
/// A registration failing mid-subscribe isolates that one tag (#739 H11): the first registration
/// stays live, the second is kept as a dead intent and goes Bad, and the call does not throw.
/// </summary>
[Fact]
public async Task Native_subscribe_failure_mid_registration_cleans_up_partial_state()
public async Task Native_subscribe_failure_mid_registration_isolates_the_failed_tag()
{
// Fail-on-second-call fake — first AddNotificationAsync succeeds, second throws.
// Subscribe's catch block must tear the first one down before rethrowing so no zombie
// notification lingers.
var fake = new FailAfterNAddsFake(new AbTagParamsIrrelevant(), succeedBefore: 1);
var factory = new FakeTwinCATClientFactory { Customise = () => fake };
var drv = new TwinCATDriver(new TwinCATDriverOptions
@@ -120,12 +121,19 @@ public sealed class TwinCATNativeNotificationTests
}, "drv-1", factory);
await drv.InitializeAsync("{}", CancellationToken.None);
await Should.ThrowAsync<InvalidOperationException>(() =>
drv.SubscribeAsync(["A", "B"], TimeSpan.FromMilliseconds(100), CancellationToken.None));
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
var handle = await drv.SubscribeAsync(["A", "B"], TimeSpan.FromMilliseconds(100), CancellationToken.None);
// First registration succeeded then got torn down by the catch; second threw.
fake.AddCallCount.ShouldBe(2);
fake.Notifications.Count.ShouldBe(0); // partial handle cleaned up
fake.Notifications.ShouldHaveSingleItem().SymbolPath.ShouldBe("MAIN.A");
var bad = events.ShouldHaveSingleItem();
bad.FullReference.ShouldBe("B");
bad.Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError);
await drv.UnsubscribeAsync(handle, CancellationToken.None);
fake.Notifications.ShouldBeEmpty();
}
/// <summary>
@@ -0,0 +1,104 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>
/// #739 P10: a bad tag warns ONCE (not once per poll — 50 typo'd symbols on a 500 ms poll used to be 100
/// Warnings/s forever), re-arms when it reads Good, and a device refusal leaves driver health alone while a
/// comms-class status degrades.
/// </summary>
[Trait("Category", "Unit")]
public sealed class TwinCATReadWarningTests
{
private const string Host = "ads://5.23.91.23.1.1:851";
private static (TwinCATDriver drv, FakeTwinCATClient fake, TwinCATCapturingLogger log) NewDriver()
{
var fake = new FakeTwinCATClient();
var log = new TwinCATCapturingLogger();
var drv = new TwinCATDriver(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host)],
RawTags = TwinCATRawTags.Entries(
[
new TwinCATTagDefinition("Typo", Host, "MAIN.Typo", TwinCATDataType.DInt),
new TwinCATTagDefinition("Other", Host, "MAIN.Other", TwinCATDataType.DInt),
]),
Probe = new TwinCATProbeOptions { Enabled = false },
}, "drv-1", new FakeTwinCATClientFactory { Customise = () => fake }, log);
return (drv, fake, log);
}
private static int WarningsFor(TwinCATCapturingLogger log, string reference) =>
log.Warnings.Count(w => w.Contains($"'{reference}'", StringComparison.Ordinal));
/// <summary>Repeated bad reads of one tag log one Warning; a second bad tag gets its own (positive control).</summary>
[Fact]
public async Task A_bad_tag_warns_once_across_polls_and_rearms_after_Good()
{
var (drv, fake, log) = NewDriver();
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
fake.ReadStatuses["MAIN.Typo"] = TwinCATStatusMapper.BadNodeIdUnknown;
for (var i = 0; i < 5; i++)
(await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken))
.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
WarningsFor(log, "Typo").ShouldBe(1);
fake.ReadStatuses["MAIN.Other"] = TwinCATStatusMapper.BadNodeIdUnknown;
_ = await drv.ReadAsync(["Other"], TestContext.Current.CancellationToken);
WarningsFor(log, "Other").ShouldBe(1);
// Fixed, then broken again: a new episode warns again.
fake.ReadStatuses.Remove("MAIN.Typo");
_ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken);
fake.ReadStatuses["MAIN.Typo"] = TwinCATStatusMapper.BadNodeIdUnknown;
_ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken);
_ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken);
WarningsFor(log, "Typo").ShouldBe(2);
}
/// <summary>A re-initialise re-arms the warning.</summary>
[Fact]
public async Task A_reinitialise_rearms_the_warning()
{
var (drv, fake, log) = NewDriver();
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
fake.ReadStatuses["MAIN.Typo"] = TwinCATStatusMapper.BadNodeIdUnknown;
_ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken);
WarningsFor(log, "Typo").ShouldBe(2);
}
/// <summary>
/// A status proving the device answered (symbol not found) keeps the driver Healthy and advances the
/// stamp; a comms-class status (timeout) degrades with the stamp preserved.
/// </summary>
[Fact]
public async Task A_refusal_status_does_not_degrade_but_a_comms_status_does()
{
var (drv, fake, _) = NewDriver();
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead!.Value;
fake.ReadStatuses["MAIN.Typo"] = TwinCATStatusMapper.BadNodeIdUnknown;
var deadline = DateTime.UtcNow.AddSeconds(2);
while (DateTime.UtcNow <= initStamp && DateTime.UtcNow < deadline) await Task.Delay(5, TestContext.Current.CancellationToken);
_ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken);
var afterRefusal = drv.GetHealth();
afterRefusal.State.ShouldBe(DriverState.Healthy);
afterRefusal.LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp);
fake.ReadStatuses["MAIN.Other"] = TwinCATStatusMapper.BadTimeout;
_ = await drv.ReadAsync(["Other"], TestContext.Current.CancellationToken);
var afterTimeout = drv.GetHealth();
afterTimeout.State.ShouldBe(DriverState.Degraded);
afterTimeout.LastSuccessfulRead.ShouldBe(afterRefusal.LastSuccessfulRead);
}
}
@@ -296,23 +296,23 @@ public sealed class TwinCATReadWriteTests
factory.Clients[0].DisposeCount.ShouldBe(1);
}
// ---- BOOL-within-word RMW writes ----
// ---- BOOL-within-word RMW writes (#739 H18: at the parent's own width, on raw bytes) ----
/// <summary>A BOOL-within-word set reads the parent word, ORs the bit, writes it back as UDInt.</summary>
/// <summary>A BOOL-within-word set read-modify-writes the parent word's raw bytes through WriteBitAsync.</summary>
[Fact]
public async Task Bit_set_RMWs_parent_word_as_UDInt()
public async Task Bit_set_RMWs_the_parent_word_bytes()
{
var (drv, factory) = NewDriver(
new TwinCATTagDefinition("Flag", "ads://5.23.91.23.1.1:851", "MAIN.Flags.3", TwinCATDataType.Bool));
factory.Customise = () => new FakeTwinCATClient { Values = { ["MAIN.Flags"] = 0b0001u } };
factory.Customise = () => new FakeTwinCATClient { ParentWords = { ["MAIN.Flags"] = [0b0001, 0, 0, 0] } };
await drv.InitializeAsync("{}", CancellationToken.None);
var results = await drv.WriteAsync([new WriteRequest("Flag", true)], CancellationToken.None);
results.Single().StatusCode.ShouldBe(TwinCATStatusMapper.Good);
factory.Clients[0].Values["MAIN.Flags"].ShouldBe(0b1001u);
factory.Clients[0].WriteLog.ShouldContain(e =>
e.symbol == "MAIN.Flags" && e.type == TwinCATDataType.UDInt && e.bit == null);
factory.Clients[0].ParentWords["MAIN.Flags"].ShouldBe(new byte[] { 0b1001, 0, 0, 0 });
factory.Clients[0].BitWriteLog.ShouldBe([("MAIN.Flags", 3, true)]);
factory.Clients[0].WriteLog.ShouldBeEmpty(); // no 32-bit UDInt round-trip any more
}
/// <summary>A BOOL-within-word clear preserves the other bits in the parent word.</summary>
@@ -321,12 +321,12 @@ public sealed class TwinCATReadWriteTests
{
var (drv, factory) = NewDriver(
new TwinCATTagDefinition("Flag", "ads://5.23.91.23.1.1:851", "MAIN.Flags.3", TwinCATDataType.Bool));
factory.Customise = () => new FakeTwinCATClient { Values = { ["MAIN.Flags"] = 0xFFFFu } };
factory.Customise = () => new FakeTwinCATClient { ParentWords = { ["MAIN.Flags"] = [0xFF, 0xFF] } };
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.WriteAsync([new WriteRequest("Flag", false)], CancellationToken.None);
factory.Clients[0].Values["MAIN.Flags"].ShouldBe(0xFFF7u);
factory.Clients[0].ParentWords["MAIN.Flags"].ShouldBe(new byte[] { 0xF7, 0xFF });
}
/// <summary>RMW works on a DWORD parent (bit 20 set above the 16-bit boundary).</summary>
@@ -335,12 +335,31 @@ public sealed class TwinCATReadWriteTests
{
var (drv, factory) = NewDriver(
new TwinCATTagDefinition("Hi", "ads://5.23.91.23.1.1:851", "GVL.Status.20", TwinCATDataType.Bool));
factory.Customise = () => new FakeTwinCATClient { Values = { ["GVL.Status"] = 0u } };
factory.Customise = () => new FakeTwinCATClient { ParentWords = { ["GVL.Status"] = [0, 0, 0, 0] } };
await drv.InitializeAsync("{}", CancellationToken.None);
await drv.WriteAsync([new WriteRequest("Hi", true)], CancellationToken.None);
factory.Clients[0].Values["GVL.Status"].ShouldBe(1u << 20);
BitConverter.ToUInt32(factory.Clients[0].ParentWords["GVL.Status"]).ShouldBe(1u << 20);
}
/// <summary>
/// A bit beyond a 16-bit parent is refused BadOutOfRange and nothing is written — before, the hardcoded
/// 32-bit RMW wrote into the neighbouring bytes. No health write: a refusal is not a comms fault.
/// </summary>
[Fact]
public async Task Bit_beyond_a_16_bit_parent_is_refused_without_writing()
{
var (drv, factory) = NewDriver(
new TwinCATTagDefinition("Hi", "ads://5.23.91.23.1.1:851", "MAIN.Word.20", TwinCATDataType.Bool));
factory.Customise = () => new FakeTwinCATClient { ParentWords = { ["MAIN.Word"] = [0, 0] } };
await drv.InitializeAsync("{}", CancellationToken.None);
var results = await drv.WriteAsync([new WriteRequest("Hi", true)], CancellationToken.None);
results.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange);
factory.Clients[0].ParentWords["MAIN.Word"].ShouldBe(new byte[] { 0, 0 });
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
/// <summary>A failed parent read short-circuits the RMW and surfaces the read status.</summary>
@@ -351,6 +370,7 @@ public sealed class TwinCATReadWriteTests
new TwinCATTagDefinition("Flag", "ads://5.23.91.23.1.1:851", "MAIN.Flags.3", TwinCATDataType.Bool));
factory.Customise = () => new FakeTwinCATClient
{
ParentWords = { ["MAIN.Flags"] = [0xAA, 0] },
ReadStatuses = { ["MAIN.Flags"] = TwinCATStatusMapper.BadNodeIdUnknown },
};
await drv.InitializeAsync("{}", CancellationToken.None);
@@ -358,25 +378,24 @@ public sealed class TwinCATReadWriteTests
var results = await drv.WriteAsync([new WriteRequest("Flag", true)], CancellationToken.None);
results.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
factory.Clients[0].WriteLog.ShouldBeEmpty();
factory.Clients[0].ParentWords["MAIN.Flags"].ShouldBe(new byte[] { 0xAA, 0 });
}
/// <summary>A Good parent read that yields a null value must NOT zero the word — it surfaces
/// a Bad status and writes nothing (treating null as 0 would clear every set bit).</summary>
/// <summary>A Good parent read that yields no bytes must NOT zero the word — it surfaces a Bad status
/// and writes nothing (treating it as 0 would clear every set bit).</summary>
[Fact]
public async Task Bit_write_with_null_parent_value_does_not_zero_the_word()
public async Task Bit_write_with_no_parent_bytes_does_not_zero_the_word()
{
var (drv, factory) = NewDriver(
new TwinCATTagDefinition("Flag", "ads://5.23.91.23.1.1:851", "MAIN.Flags.3", TwinCATDataType.Bool));
// Do NOT seed Values["MAIN.Flags"] — the fake returns (null, Good) for the parent read.
// Do NOT seed ParentWords["MAIN.Flags"] — the fake answers the parent read Good with no bytes.
factory.Customise = () => new FakeTwinCATClient();
await drv.InitializeAsync("{}", CancellationToken.None);
var results = await drv.WriteAsync([new WriteRequest("Flag", true)], CancellationToken.None);
results.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError);
// No parent write attempted — the word is left untouched on the device.
factory.Clients[0].WriteLog.ShouldBeEmpty();
factory.Clients[0].ParentWords.ShouldNotContainKey("MAIN.Flags");
}
/// <summary>Concurrent bit writes to the same word compose correctly (per-parent lock).</summary>
@@ -387,12 +406,12 @@ public sealed class TwinCATReadWriteTests
.Select(b => new TwinCATTagDefinition($"Bit{b}", "ads://5.23.91.23.1.1:851", $"MAIN.Flags.{b}", TwinCATDataType.Bool))
.ToArray();
var (drv, factory) = NewDriver(tags);
factory.Customise = () => new FakeTwinCATClient { Values = { ["MAIN.Flags"] = 0u } };
factory.Customise = () => new FakeTwinCATClient { YieldInBitWrite = true, ParentWords = { ["MAIN.Flags"] = [0, 0] } };
await drv.InitializeAsync("{}", CancellationToken.None);
await Task.WhenAll(Enumerable.Range(0, 8).Select(b =>
drv.WriteAsync([new WriteRequest($"Bit{b}", true)], CancellationToken.None)));
Convert.ToUInt32(factory.Clients[0].Values["MAIN.Flags"]).ShouldBe(0xFFu);
factory.Clients[0].ParentWords["MAIN.Flags"].ShouldBe(new byte[] { 0xFF, 0 });
}
}
@@ -0,0 +1,302 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Shouldly;
using TwinCAT.TypeSystem;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>
/// #739 review fix round 1: the PLC-not-in-RUN health ruling, refused-registration retry backoff,
/// per-device worst-of health, RMW gates surviving teardown, a cancellable shutdown, the real
/// STRING / array encode path through the client planning helpers, and REAL range checks.
/// </summary>
[Trait("Category", "Unit")]
public sealed class TwinCATReviewFixRound1Tests
{
private const string Host = "ads://5.23.91.23.1.1:851";
private const string HostB = "ads://5.23.91.24.1.1:851";
private static TwinCATProbeOptions FastProbe => new()
{
Enabled = true,
Interval = TimeSpan.FromMilliseconds(30),
Timeout = TimeSpan.FromMilliseconds(100),
};
private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, TwinCATProbeOptions probe, bool native,
params TwinCATTagDefinition[] tags) =>
new(new TwinCATDriverOptions
{
Devices = [.. tags.Select(t => t.DeviceHostAddress).Append(Host).Distinct().Select(h => new TwinCATDeviceOptions(h))],
RawTags = TwinCATRawTags.Entries(tags),
Probe = probe,
UseNativeNotifications = native,
}, "drv-1", factory);
private static async Task WaitForAsync(Func<bool> condition, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (!condition() && DateTime.UtcNow < deadline)
await Task.Delay(10);
}
// ---- IMPORTANT: a PLC not in RUN leaves the driver Degraded ----
/// <summary>
/// A probe that answers with ADS state Stop writes Degraded "PLC in Stop" and still advances
/// LastSuccessfulRead (the device answered); back in Run the driver is Healthy (positive control).
/// </summary>
[Fact]
public async Task A_PLC_not_in_Run_is_Degraded_and_Run_restores_Healthy()
{
var fake = new FakeTwinCATClient { AdsStateText = "Stop" };
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: false,
new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead!.Value;
await WaitForAsync(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(3));
var stopped = drv.GetHealth();
stopped.State.ShouldBe(DriverState.Degraded);
stopped.LastError.ShouldNotBeNull().ShouldContain("PLC in Stop");
await WaitForAsync(() => drv.GetHealth().LastSuccessfulRead > initStamp, TimeSpan.FromSeconds(3));
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp);
// A Good read while the PLC is stopped must not paper over it.
_ = await drv.ReadAsync(["Speed"], TestContext.Current.CancellationToken);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
fake.AdsStateText = "Run";
await WaitForAsync(() => drv.GetHealth().State == DriverState.Healthy, TimeSpan.FromSeconds(3));
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
// ---- MINOR 1: refused dead registrations back off; a class change republishes ----
/// <summary>
/// A REFUSED dead registration is not re-attempted every probe tick; a comms-failed one is (positive
/// control: it recovers). A symbol-version-changed signal re-arms the refused one at once.
/// </summary>
[Fact]
public async Task Refused_registrations_back_off_while_comms_failures_retry_each_tick()
{
var fake = new FakeTwinCATClient();
fake.AddNotificationErrors["MAIN.T"] = new TwinCATTypeMismatchException("MAIN.T", "declared DInt, PLC STRING");
fake.AddNotificationErrors["MAIN.C"] = new InvalidOperationException("transient");
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: true,
new TwinCATTagDefinition("T", Host, "MAIN.T", TwinCATDataType.DInt),
new TwinCATTagDefinition("C", Host, "MAIN.C", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.SubscribeAsync(["T", "C"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
await WaitForAsync(() => fake.AttemptsFor("MAIN.C") >= 3, TimeSpan.FromSeconds(3));
fake.AttemptsFor("MAIN.C").ShouldBeGreaterThanOrEqualTo(3); // comms class: every tick
fake.AttemptsFor("MAIN.T").ShouldBe(1); // refusal: backed off
fake.AddNotificationErrors.TryRemove("MAIN.C", out _);
await WaitForAsync(() => fake.Notifications.Any(n => n.SymbolPath == "MAIN.C"), TimeSpan.FromSeconds(3));
fake.Notifications.ShouldContain(n => n.SymbolPath == "MAIN.C");
fake.FireSymbolVersionChanged();
await WaitForAsync(() => fake.AttemptsFor("MAIN.T") >= 2, TimeSpan.FromSeconds(3));
fake.AttemptsFor("MAIN.T").ShouldBe(2);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
/// <summary>When a retry fails with a different failure class the tag's status is republished, once.</summary>
[Fact]
public async Task A_retry_whose_failure_class_changes_republishes_the_status()
{
var fake = new FakeTwinCATClient();
fake.AddNotificationErrors["MAIN.X"] = new InvalidOperationException("router hiccup");
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: true,
new TwinCATTagDefinition("X", Host, "MAIN.X", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
_ = await drv.SubscribeAsync(["X"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
events.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError);
await WaitForAsync(() => fake.AttemptsFor("MAIN.X") >= 3, TimeSpan.FromSeconds(3));
events.Count.ShouldBe(1, "a repeat of the same failure class publishes nothing");
fake.AddNotificationErrors["MAIN.X"] = new TwinCATNotificationRefusedException(
"MAIN.X", "symbol gone", TwinCATStatusMapper.BadNodeIdUnknown);
await WaitForAsync(() => events.Count >= 2, TimeSpan.FromSeconds(3));
events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
// ---- MINOR 2: per-device worst-of health ----
/// <summary>
/// A comms fault on device B is not cleared by device A answering; B answering clears it
/// (positive control).
/// </summary>
[Fact]
public async Task One_devices_answer_does_not_clear_another_devices_comms_fault()
{
var clients = new ConcurrentDictionary<int, FakeTwinCATClient>();
var build = 0;
var factory = new FakeTwinCATClientFactory
{
Customise = () =>
{
var c = new FakeTwinCATClient();
clients[Interlocked.Increment(ref build)] = c;
return c;
},
};
var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false }, native: false,
new TwinCATTagDefinition("A", Host, "MAIN.A", TwinCATDataType.DInt),
new TwinCATTagDefinition("B", HostB, "MAIN.B", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); // client 1 = device A
_ = await drv.ReadAsync(["B"], TestContext.Current.CancellationToken); // client 2 = device B
clients[2].ReadStatuses["MAIN.B"] = TwinCATStatusMapper.BadTimeout;
_ = await drv.ReadAsync(["B"], TestContext.Current.CancellationToken);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
_ = await drv.ReadAsync(["A"], TestContext.Current.CancellationToken);
drv.GetHealth().State.ShouldBe(DriverState.Degraded, "device A answering says nothing about device B");
clients[2].ReadStatuses.Remove("MAIN.B");
_ = await drv.ReadAsync(["B"], TestContext.Current.CancellationToken);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
// ---- MINOR 3: RMW gates are kept across generations ----
/// <summary>The per-parent RMW gate survives a teardown (a write still in flight keeps the same gate).</summary>
[Fact]
public async Task Bit_RMW_gates_survive_a_shutdown()
{
var fake = new FakeTwinCATClient { ParentWords = { ["MAIN.Flags"] = [0, 0] } };
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake },
new TwinCATProbeOptions { Enabled = false }, native: false,
new TwinCATTagDefinition("Bit", Host, "MAIN.Flags.1", TwinCATDataType.Bool));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.WriteAsync([new WriteRequest("Bit", true)], TestContext.Current.CancellationToken);
var gate = drv.BitRmwGateForTest(Host, "MAIN.Flags").ShouldNotBeNull();
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.WriteAsync([new WriteRequest("Bit", false)], TestContext.Current.CancellationToken);
drv.BitRmwGateForTest(Host, "MAIN.Flags").ShouldBeSameAs(gate);
}
// ---- MINOR 4: ShutdownAsync honours its token ----
/// <summary>
/// A cancelled ShutdownAsync stops waiting on a probe wedged in connect and throws
/// OperationCanceledException; the teardown itself still happened (tables emptied).
/// </summary>
[Fact]
public async Task Shutdown_honours_its_cancellation_token()
{
var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { BlockConnect = wedge } };
var drv = NewDriver(factory, FastProbe, native: false);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2));
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(100));
var sw = Stopwatch.StartNew();
await Should.ThrowAsync<OperationCanceledException>(() => drv.ShutdownAsync(cts.Token));
sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ProbeJoinTimeout, "the wait ended on the token, not the join bound");
drv.DeviceCount.ShouldBe(0);
drv.GetHealth().State.ShouldBe(DriverState.Unknown);
// The background teardown finishes once the wedge releases, and disposes the client the wedged
// connect produced (no dangling teardown left behind the test).
var teardown = drv.LastTeardownForTest.ShouldNotBeNull();
wedge.SetResult();
await teardown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
await WaitForAsync(() => factory.Clients[0].DisposeCount > 0, TimeSpan.FromSeconds(3));
factory.Clients[0].DisposeCount.ShouldBeGreaterThan(0);
}
// ---- MINOR 7: the real STRING / array encode path through the client planning helpers ----
/// <summary>
/// A STRING write through the driver lands as NUL-padded Windows-1252 bytes at the PLC symbol's width;
/// an over-long value is BadOutOfRange and a WSTRING symbol under a String tag BadTypeMismatch.
/// </summary>
[Fact]
public async Task String_writes_go_through_the_client_encode_plan()
{
var fake = new FakeTwinCATClient();
fake.SymbolShapes["MAIN.Name"] = TwinCATPlcSymbolShape.Classify("STRING(4)", DataTypeCategory.String, 5);
fake.SymbolShapes["MAIN.Wide"] = TwinCATPlcSymbolShape.Classify("WSTRING(4)", DataTypeCategory.String, 10);
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake },
new TwinCATProbeOptions { Enabled = false }, native: false,
new TwinCATTagDefinition("Name", Host, "MAIN.Name", TwinCATDataType.String),
new TwinCATTagDefinition("Wide", Host, "MAIN.Wide", TwinCATDataType.String));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var results = await drv.WriteAsync(
[
new WriteRequest("Name", "äb"),
new WriteRequest("Name", "toolong"),
new WriteRequest("Wide", "x"),
], TestContext.Current.CancellationToken);
results.Select(r => r.StatusCode).ShouldBe(
[TwinCATStatusMapper.Good, TwinCATStatusMapper.BadOutOfRange, TwinCATStatusMapper.BadTypeMismatch]);
fake.RawWrites["MAIN.Name"].ShouldBe(new byte[] { 0xE4, (byte)'b', 0, 0, 0 });
fake.RawWrites.ShouldNotContainKey("MAIN.Wide");
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
/// <summary>An array write through the driver lands as the codec's bytes at the PLC element stride.</summary>
[Fact]
public async Task Array_writes_go_through_the_client_encode_plan()
{
var fake = new FakeTwinCATClient();
var element = TwinCATPlcSymbolShape.Classify("INT", DataTypeCategory.Primitive, 2);
fake.SymbolShapes["MAIN.Buf"] = TwinCATPlcSymbolShape.Classify("ARRAY [0..3] OF INT", DataTypeCategory.Array, 8);
fake.ArrayShapes["MAIN.Buf"] = new TwinCATPlcArrayShape([4], 8, element);
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake },
new TwinCATProbeOptions { Enabled = false }, native: false,
new TwinCATTagDefinition("Buf", Host, "MAIN.Buf", TwinCATDataType.Int, ArrayLength: 2),
new TwinCATTagDefinition("BufAsReal", Host, "MAIN.Buf", TwinCATDataType.Real, ArrayLength: 2));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var results = await drv.WriteAsync(
[
new WriteRequest("Buf", new[] { 1, -2 }),
new WriteRequest("BufAsReal", new[] { 1.0, 2.0 }),
], TestContext.Current.CancellationToken);
results[0].StatusCode.ShouldBe(TwinCATStatusMapper.Good);
fake.RawWrites["MAIN.Buf"].ShouldBe(new byte[] { 0x01, 0x00, 0xFE, 0xFF }); // leading sub-range only
results[1].StatusCode.ShouldBe(TwinCATStatusMapper.BadTypeMismatch);
}
// ---- MINOR 8: REAL / LREAL range ----
/// <summary>A REAL (scalar or array element) beyond float range is BadOutOfRange, never written as Infinity.</summary>
[Fact]
public async Task Out_of_range_REAL_values_are_BadOutOfRange()
{
Should.Throw<OverflowException>(() => AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, 1e300));
Should.Throw<OverflowException>(() => AdsTwinCATClient.ConvertForWrite(TwinCATDataType.LReal, "1e400"));
AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, 3.5).ShouldBe(3.5f);
AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, float.PositiveInfinity).ShouldBe(float.PositiveInfinity);
float.IsNaN((float)AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, double.NaN)).ShouldBeTrue();
var fake = new FakeTwinCATClient();
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake },
new TwinCATProbeOptions { Enabled = false }, native: false,
new TwinCATTagDefinition("Arr", Host, "MAIN.Arr", TwinCATDataType.Real, ArrayLength: 2));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var result = await drv.WriteAsync([new WriteRequest("Arr", new[] { 1.0, 1e300 })], TestContext.Current.CancellationToken);
result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange);
fake.ArrayWriteLog.ShouldBeEmpty();
}
}
@@ -0,0 +1,262 @@
using System.Collections.Concurrent;
using Shouldly;
using TwinCAT.TypeSystem;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>
/// #739 review fix round 2: a cancelled shutdown's background teardown is confined to its own generation
/// (I1); write outcomes feed per-device health (M1); a symbol-version re-arm survives an in-flight retry
/// and restarts the backoff (M2/M3); WSTRING / multi-dimensional encode plans (M5); infinity strings (M6).
/// </summary>
[Trait("Category", "Unit")]
public sealed class TwinCATReviewFixRound2Tests
{
private const string Host = "ads://5.23.91.23.1.1:851";
private static TwinCATProbeOptions FastProbe => new()
{
Enabled = true,
Interval = TimeSpan.FromMilliseconds(30),
Timeout = TimeSpan.FromMilliseconds(100),
};
private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, TwinCATProbeOptions probe, bool native,
params TwinCATTagDefinition[] tags) =>
new(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host)],
RawTags = TwinCATRawTags.Entries(tags),
Probe = probe,
UseNativeNotifications = native,
}, "drv-1", factory);
private static async Task WaitForAsync(Func<bool> condition, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (!condition() && DateTime.UtcNow < deadline)
await Task.Delay(10);
}
// ---- I1 ----
/// <summary>
/// The exact reviewed sequence: a poll read is wedged; ShutdownAsync is cancelled (its teardown
/// continues in the background); the same instance is re-initialised and subscribed; the wedge is
/// released and the OLD teardown completes. The NEW subscription keeps polling and delivering (positive
/// control), UnsubscribeAsync finds and stops it, and a final shutdown leaves nothing polling.
/// </summary>
[Fact]
public async Task A_cancelled_shutdowns_background_teardown_never_touches_the_next_generations_subscription()
{
var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var counter = 0;
var clients = new ConcurrentQueue<FakeTwinCATClient>();
var factory = new FakeTwinCATClientFactory
{
Customise = () =>
{
// Client 1 (generation 1) wedges its first read; later clients answer an ever-changing value
// so every poll publishes.
var c = new ChangingValueFake(() => Interlocked.Increment(ref counter))
{
BlockRead = clients.IsEmpty ? wedge : null,
};
clients.Enqueue(c);
return c;
},
};
var tag = new TwinCATTagDefinition("A", Host, "MAIN.A", TwinCATDataType.DInt);
var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false }, native: false, tag);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
_ = await drv.SubscribeAsync(["A"], TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken);
await WaitForAsync(() => !clients.IsEmpty, TimeSpan.FromSeconds(2)); // gen-1 poll read is wedged
using (var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)))
await Should.ThrowAsync<OperationCanceledException>(() => drv.ShutdownAsync(cts.Token));
var oldTeardown = drv.LastTeardownForTest.ShouldNotBeNull();
oldTeardown.IsCompleted.ShouldBeFalse("the old teardown must still be running in the background");
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var handle = await drv.SubscribeAsync(["A"], TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken);
wedge.SetResult();
await oldTeardown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
// Positive control: the new subscription delivers AFTER the old teardown finished.
var afterTeardown = events.Count(e => e.SubscriptionHandle.Equals(handle));
await WaitForAsync(() => events.Count(e => e.SubscriptionHandle.Equals(handle)) >= afterTeardown + 3, TimeSpan.FromSeconds(3));
events.Count(e => e.SubscriptionHandle.Equals(handle)).ShouldBeGreaterThanOrEqualTo(afterTeardown + 3);
// UnsubscribeAsync can still find and stop it.
await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken);
var stoppedAt = events.Count(e => e.SubscriptionHandle.Equals(handle));
// Measured window: a sibling driver's poll publishes 5 times; the stopped loop must publish nothing.
var witnessFactory = new FakeTwinCATClientFactory { Customise = () => new ChangingValueFake(() => Environment.TickCount) };
var witnessDrv = NewDriver(witnessFactory, new TwinCATProbeOptions { Enabled = false }, native: false, tag);
await witnessDrv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var witnessEvents = 0;
witnessDrv.OnDataChange += (_, _) => Interlocked.Increment(ref witnessEvents);
_ = await witnessDrv.SubscribeAsync(["A"], TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken);
await WaitForAsync(() => Volatile.Read(ref witnessEvents) >= 5, TimeSpan.FromSeconds(3));
Volatile.Read(ref witnessEvents).ShouldBeGreaterThanOrEqualTo(5);
await witnessDrv.ShutdownAsync(TestContext.Current.CancellationToken);
events.Count(e => e.SubscriptionHandle.Equals(handle)).ShouldBe(stoppedAt, "the unsubscribed loop must have stopped");
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
/// <summary>Answers each read with a new value so every poll cycle publishes.</summary>
private sealed class ChangingValueFake(Func<int> next) : FakeTwinCATClient
{
public override Task<(object? value, uint status)> ReadValueAsync(
string symbolPath, TwinCATDataType type, int? bitIndex, int? arrayCount, CancellationToken ct)
{
if (BlockRead is not null) return base.ReadValueAsync(symbolPath, type, bitIndex, arrayCount, ct);
Values[symbolPath] = next();
return base.ReadValueAsync(symbolPath, type, bitIndex, arrayCount, ct);
}
}
// ---- M1 ----
/// <summary>
/// A write-only device with the probe off: a wire-failed write degrades; a later Good write is the device
/// answering — Healthy again with the stamp advanced. A refused write (BadNotWritable) changes nothing.
/// </summary>
[Fact]
public async Task A_good_write_clears_that_devices_comms_fault()
{
var fake = new FakeTwinCATClient();
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake },
new TwinCATProbeOptions { Enabled = false }, native: false,
new TwinCATTagDefinition("W", Host, "MAIN.W", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead!.Value;
fake.ThrowOnWrite = true;
_ = await drv.WriteAsync([new WriteRequest("W", 1)], TestContext.Current.CancellationToken);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
fake.ThrowOnWrite = false;
fake.WriteStatuses["MAIN.W"] = TwinCATStatusMapper.BadNotWritable;
_ = await drv.WriteAsync([new WriteRequest("W", 1)], TestContext.Current.CancellationToken);
drv.GetHealth().State.ShouldBe(DriverState.Degraded, "a refusal says nothing about the comms fault");
await WaitForAsync(() => DateTime.UtcNow > initStamp, TimeSpan.FromSeconds(1));
fake.WriteStatuses.Remove("MAIN.W");
_ = await drv.WriteAsync([new WriteRequest("W", 1)], TestContext.Current.CancellationToken);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp);
}
// ---- M2 + M3 ----
/// <summary>
/// A symbol-version re-arm that lands while a retry of the refused registration is in flight is not
/// overwritten by that retry's failure: the next tick re-attempts at once. (Without the epoch the
/// failure re-imposed a 30 s+ backoff.)
/// </summary>
[Fact]
public async Task A_rearm_during_an_in_flight_retry_is_not_lost()
{
var fake = new FakeTwinCATClient();
fake.AddNotificationErrors["MAIN.T"] = new TwinCATTypeMismatchException("MAIN.T", "PLC STRING");
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: true,
new TwinCATTagDefinition("T", Host, "MAIN.T", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.SubscribeAsync(["T"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
fake.AttemptsFor("MAIN.T").ShouldBe(1);
var inFlight = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
fake.PreAttemptGate = inFlight;
fake.FireSymbolVersionChanged(); // re-arm: the next tick starts a retry, which wedges on the gate
await WaitForAsync(() => fake.AttemptsFor("MAIN.T") >= 2, TimeSpan.FromSeconds(3));
fake.FireSymbolVersionChanged(); // a second program change while that retry is in flight
fake.PreAttemptGate = null;
inFlight.SetResult(); // the in-flight retry now fails (still refused)
await WaitForAsync(() => fake.AttemptsFor("MAIN.T") >= 3, TimeSpan.FromSeconds(3));
fake.AttemptsFor("MAIN.T").ShouldBeGreaterThanOrEqualTo(3);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
/// <summary>A re-arm restarts the refusal backoff from the initial delay instead of doubling the old one.</summary>
[Fact]
public async Task A_rearm_restarts_the_refusal_backoff()
{
var fake = new FakeTwinCATClient();
fake.AddNotificationErrors["MAIN.T"] = new TwinCATTypeMismatchException("MAIN.T", "PLC STRING");
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: true,
new TwinCATTagDefinition("T", Host, "MAIN.T", TwinCATDataType.DInt));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.SubscribeAsync(["T"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
var reg = drv.GetDeviceState(Host)!.NativeRegistrations.Values.Single();
reg.RefusalBackoff.ShouldBe(TwinCATDriver.RefusedRetryInitial);
fake.FireSymbolVersionChanged();
await WaitForAsync(() => fake.AttemptsFor("MAIN.T") >= 2, TimeSpan.FromSeconds(3));
await WaitForAsync(() => reg.RefusalBackoff != TimeSpan.Zero, TimeSpan.FromSeconds(1));
reg.RefusalBackoff.ShouldBe(TwinCATDriver.RefusedRetryInitial);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
// ---- M5 ----
/// <summary>A WSTRING tag over a WSTRING symbol writes UTF-16LE with an aligned 2-byte terminator, zero-padded.</summary>
[Fact]
public async Task A_WSTRING_write_goes_through_the_client_encode_plan()
{
var fake = new FakeTwinCATClient();
fake.SymbolShapes["MAIN.W"] = TwinCATPlcSymbolShape.Classify("WSTRING(3)", DataTypeCategory.String, 8);
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake },
new TwinCATProbeOptions { Enabled = false }, native: false,
new TwinCATTagDefinition("W", Host, "MAIN.W", TwinCATDataType.WString));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var results = await drv.WriteAsync(
[new WriteRequest("W", "中x"), new WriteRequest("W", "abcd")], TestContext.Current.CancellationToken);
results[0].StatusCode.ShouldBe(TwinCATStatusMapper.Good);
results[1].StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange); // 4 chars do not fit WSTRING(3)
fake.RawWrites["MAIN.W"].ShouldBe(new byte[] { 0x2D, 0x4E, (byte)'x', 0, 0, 0, 0, 0 });
}
/// <summary>A multi-dimensional array is written flattened in PLC memory order (last index fastest).</summary>
[Fact]
public async Task A_multi_dimensional_array_write_is_flattened_row_major()
{
var fake = new FakeTwinCATClient();
var element = TwinCATPlcSymbolShape.Classify("INT", DataTypeCategory.Primitive, 2);
fake.SymbolShapes["MAIN.M"] = TwinCATPlcSymbolShape.Classify("ARRAY [0..1,0..2] OF INT", DataTypeCategory.Array, 12);
fake.ArrayShapes["MAIN.M"] = new TwinCATPlcArrayShape([2, 3], 12, element);
var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake },
new TwinCATProbeOptions { Enabled = false }, native: false,
new TwinCATTagDefinition("M", Host, "MAIN.M", TwinCATDataType.Int, ArrayLength: 6));
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var result = await drv.WriteAsync([new WriteRequest("M", new[] { 1, 2, 3, 4, 5, 6 })], TestContext.Current.CancellationToken);
result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.Good);
fake.RawWrites["MAIN.M"].ShouldBe(new byte[] { 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0 });
TwinCATArrayCodec.Decode(TwinCATDataType.Int, fake.RawWrites["MAIN.M"], new TwinCATArrayLayout(6, 2), false)
.ShouldBe(new short[] { 1, 2, 3, 4, 5, 6 });
}
// ---- M6 ----
/// <summary>An infinity spelled as a string passes like a float/double infinity; a finite overflow does not.</summary>
[Fact]
public void Infinity_strings_are_explicit_infinities()
{
AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, "Infinity").ShouldBe(float.PositiveInfinity);
AdsTwinCATClient.ConvertForWrite(TwinCATDataType.LReal, "-Infinity").ShouldBe(double.NegativeInfinity);
Should.Throw<OverflowException>(() => AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, "1e300"));
}
}
@@ -31,6 +31,26 @@ public sealed class DiagnoseTests
=> Diag("""var c = new System.Net.Http.HttpClient(); return 0;""")
.ShouldContain(x => x.Severity == 8);
// Sandbox hardening parity (R05:10): the editor runs the SAME ForbiddenTypeAnalyzer the publish gate
// runs, so every newly-denied form must surface as an OTSCRIPT_FORBIDDEN marker here — editor accepts
// ⇔ publish accepts. One case per deny mechanism (namespace, member, type), plus the allow-list.
[Theory]
[InlineData("""new System.Threading.SemaphoreSlim(0).Wait(); return 0;""")] // System.Threading namespace
[InlineData("""System.Threading.SpinWait.SpinUntil(() => false); return 0;""")]
[InlineData("""var t = System.Type.GetType("System.IO.File"); return t is null ? 0 : 1;""")] // forbidden member
[InlineData("""return typeof(string).GetMethods().Length;""")]
[InlineData("""var o = System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(string)); return 0;""")] // forbidden type
public void Hardened_sandbox_forms_are_flagged_in_the_editor(string code)
=> Diag(code).ShouldContain(x => x.Code == "OTSCRIPT_FORBIDDEN" && x.Severity == 8);
[Theory]
[InlineData("""var t = System.Threading.CancellationToken.None; return t.IsCancellationRequested ? 1 : 0;""")]
[InlineData("""var x = 41; System.Threading.Interlocked.Increment(ref x); return x;""")]
[InlineData("""var x = 42; return System.Threading.Volatile.Read(ref x);""")]
[InlineData("""object o = 42; return o.GetType().IsValueType ? 1 : 0;""")]
public void Allow_listed_forms_are_not_flagged_in_the_editor(string code)
=> Diag(code).ShouldBeEmpty();
[Fact] public void Dynamic_tag_path_is_flagged()
=> Diag("""var p = "A"; return ctx.GetTag(p).Value;""")
.ShouldContain(x => x.Message.Contains("literal"));
@@ -169,4 +169,26 @@ public sealed class FocasTagConfigModelTests
[InlineData("""{"address":"R100","dataType":2}""")]
public void Validate_accepts_every_dataType_the_driver_accepts(string json)
=> FocasTagConfigModel.FromJson(json).Validate().ShouldBeNull();
/// <summary>
/// #740 / R07:U8 — String on a PMC / PARAM / MACRO address is refused by the driver (no area read
/// returns text), so the modal refuses it too, for a model built in the editor and for a loaded blob.
/// </summary>
[Theory]
[InlineData("R100")]
[InlineData("PARAM:6711")]
[InlineData("MACRO:500")]
public void Validate_refuses_string_on_an_area_address(string address)
{
new FocasTagConfigModel { Address = address, DataType = FocasDataType.String }
.Validate().ShouldNotBeNull().ShouldContain("String");
FocasTagConfigModel.FromJson($$"""{"address":"{{address}}","dataType":"String"}""")
.Validate().ShouldNotBeNull().ShouldContain("String");
}
/// <summary>Positive control: the fixed tree's string fields keep String.</summary>
[Fact]
public void Validate_accepts_string_on_a_fixed_tree_path()
=> new FocasTagConfigModel { Address = "FIXED:Identity/SeriesNumber", DataType = FocasDataType.String }
.Validate().ShouldBeNull();
}
@@ -0,0 +1,159 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// R03:3.1 — the admin singleton's mailbox used to be suspended for the whole of every
/// <c>ReceiveAsync</c>: one Test Connect against a black-holed PLC (60 s clamp) or one large deploy
/// compose stalled every other admin operation fleet-wide, alarm acks included. Both now run off the
/// actor thread; only the "is one in flight" check stays serialised.
/// </summary>
public sealed class AdminOperationsActorMailboxTests : ControlPlaneActorTestBase
{
private static readonly TimeSpan AckBudget = TimeSpan.FromSeconds(1);
/// <summary>An alarm ack completes in under a second while a 60 s Test Connect is still pending.</summary>
[Fact]
public void Alarm_ack_completes_while_a_60s_test_connect_is_pending()
{
var dbFactory = NewInMemoryDbFactory();
var coordinator = CreateTestProbe("coord");
var probe = new BlockingProbe();
var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe }));
var connectClient = CreateTestProbe("connect-client");
connectClient.Send(actor, new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 60, Guid.NewGuid()));
probe.Started.Wait(PresenceBudget).ShouldBeTrue("the probe must be in flight before the ack is sent");
actor.Tell(new AcknowledgeAlarmCommand("alarm-1", "op", null, CorrelationId.NewId()));
var ack = ExpectMsg<AcknowledgeAlarmResult>(AckBudget);
ack.Ok.ShouldBeTrue();
// Positive control: the probe reply still lands, with the right sender, once the probe returns.
connectClient.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
probe.Release(new DriverProbeResult(true, null, TimeSpan.FromMilliseconds(5)));
var result = connectClient.ExpectMsg<TestDriverConnectResult>(PresenceBudget);
result.Ok.ShouldBeTrue();
result.LatencyMs.ShouldNotBeNull();
}
/// <summary>A probe that times out still answers its own sender with the clamped timeout message.</summary>
[Fact]
public void Test_connect_timeout_is_reported_to_its_sender()
{
var dbFactory = NewInMemoryDbFactory();
var coordinator = CreateTestProbe("coord");
var probe = new BlockingProbe(); // never released: the CTS fires at the 1 s clamp floor
var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe }));
actor.Tell(new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 1, Guid.NewGuid()));
var result = ExpectMsg<TestDriverConnectResult>(PresenceBudget);
result.Ok.ShouldBeFalse();
result.Message.ShouldNotBeNull();
result.Message.ShouldContain("timed out");
}
/// <summary>
/// The deploy compose (snapshot + validate + gzip + row write) runs off the actor: an alarm ack
/// completes while the compose is parked on a gated DB context, and the deploy still completes
/// (Accepted + dispatched) once the gate opens.
/// </summary>
[Fact]
public void Alarm_ack_completes_while_a_deploy_compose_is_in_flight()
{
var gated = new GatedDbFactory(NewInMemoryDbFactory());
var coordinator = CreateTestProbe("coord");
var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty<IDriverProbe>()));
var deployer = CreateTestProbe("deployer");
gated.Hold();
deployer.Send(actor, new StartDeployment("joe", CorrelationId.NewId()));
gated.Entered.Wait(PresenceBudget).ShouldBeTrue("the compose must be parked before the ack is sent");
actor.Tell(new AcknowledgeAlarmCommand("alarm-2", "op", null, CorrelationId.NewId()));
ExpectMsg<AcknowledgeAlarmResult>(AckBudget).Ok.ShouldBeTrue();
// Positive control: the deploy completes once released.
deployer.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
gated.Release();
deployer.ExpectMsg<StartDeploymentResult>(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
coordinator.ExpectMsg<DispatchDeployment>(PresenceBudget);
}
/// <summary>
/// Only the in-flight check is serialised — and it must still catch a second request that arrives
/// while the first compose has not yet written its row (the DB alone would let both through).
/// </summary>
[Fact]
public void Second_StartDeployment_during_an_in_flight_compose_is_refused()
{
var gated = new GatedDbFactory(NewInMemoryDbFactory());
var coordinator = CreateTestProbe("coord");
var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty<IDriverProbe>()));
var first = CreateTestProbe("first");
var second = CreateTestProbe("second");
gated.Hold();
first.Send(actor, new StartDeployment("joe", CorrelationId.NewId()));
gated.Entered.Wait(PresenceBudget).ShouldBeTrue();
second.Send(actor, new StartDeployment("kim", CorrelationId.NewId()));
second.ExpectMsg<StartDeploymentResult>(AckBudget).Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight);
gated.Release();
first.ExpectMsg<StartDeploymentResult>(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
coordinator.ExpectMsg<DispatchDeployment>(PresenceBudget);
}
private sealed class BlockingProbe : IDriverProbe
{
private readonly TaskCompletionSource<DriverProbeResult> _tcs =
new(TaskCreationOptions.RunContinuationsAsynchronously);
public ManualResetEventSlim Started { get; } = new();
public string DriverType => "Blocking";
public async Task<DriverProbeResult> ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct)
{
Started.Set();
return await _tcs.Task.WaitAsync(ct);
}
public void Release(DriverProbeResult result) => _tcs.TrySetResult(result);
}
/// <summary>
/// Passes synchronous context creation through (the serialised in-flight check) and parks the
/// asynchronous one (the off-actor compose) on a gate while <see cref="Hold"/> is in effect.
/// </summary>
private sealed class GatedDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner)
: IDbContextFactory<OtOpcUaConfigDbContext>
{
private volatile TaskCompletionSource? _gate;
public ManualResetEventSlim Entered { get; } = new();
public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
public void Release() => _gate?.TrySetResult();
public OtOpcUaConfigDbContext CreateDbContext() => inner.CreateDbContext();
public async Task<OtOpcUaConfigDbContext> CreateDbContextAsync(CancellationToken cancellationToken = default)
{
Entered.Set();
if (_gate is { } gate) await gate.Task.WaitAsync(cancellationToken);
return inner.CreateDbContext();
}
}
}
@@ -0,0 +1,68 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// R03:4.1 — admin operations echoed <c>ex.Message</c> to the UI, so a SqlException's connection-string
/// fragment reached any ConfigEditor browser session. The reply is now a fixed sentence plus the request's
/// correlation id; the detail stays in the central log.
/// </summary>
public sealed class AdminOperationsActorServerFaultMessageTests : ControlPlaneActorTestBase
{
private const string Secret = "Server=sql-prod-01;Password=hunter2";
[Fact]
public void StartDeployment_server_fault_reply_carries_the_correlation_id_and_not_the_exception_text()
{
var coordinator = CreateTestProbe("coord");
var actor = Sys.ActorOf(AdminOperationsActor.Props(new ThrowingDbFactory(), coordinator.Ref, Enumerable.Empty<IDriverProbe>()));
var correlationId = CorrelationId.NewId();
actor.Tell(new StartDeployment("joe", correlationId));
var reply = ExpectMsg<StartDeploymentResult>(PresenceBudget);
reply.Outcome.ShouldBe(StartDeploymentOutcome.Rejected);
reply.Message.ShouldNotBeNull();
reply.Message.ShouldNotContain("hunter2");
reply.Message.ShouldNotContain("sql-prod-01");
reply.Message.ShouldContain(correlationId.ToString()); // the operator can find the log line
}
[Fact]
public void TestDriverConnect_probe_throw_reply_carries_the_correlation_id_and_not_the_exception_text()
{
var coordinator = CreateTestProbe("coord");
var actor = Sys.ActorOf(AdminOperationsActor.Props(NewInMemoryDbFactory(), coordinator.Ref, new IDriverProbe[] { new ThrowingProbe() }));
var correlationId = Guid.NewGuid();
actor.Tell(new TestDriverConnect("Throwing", "{}", 5, correlationId));
var reply = ExpectMsg<TestDriverConnectResult>(PresenceBudget);
reply.Ok.ShouldBeFalse();
reply.Message.ShouldNotBeNull();
reply.Message.ShouldNotContain("hunter2");
reply.Message.ShouldContain(correlationId.ToString());
}
private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
{
public OtOpcUaConfigDbContext CreateDbContext() =>
throw new InvalidOperationException("Cannot open database: " + Secret);
}
private sealed class ThrowingProbe : IDriverProbe
{
public string DriverType => "Throwing";
public Task<DriverProbeResult> ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) =>
throw new InvalidOperationException("probe blew up: " + Secret);
}
}
@@ -0,0 +1,133 @@
using System.IO.Compression;
using System.Text;
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
using Microsoft.EntityFrameworkCore.Storage;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// R03:3.2 — <c>StartDeployment</c> used to read the whole config DB twice: once for the validator,
/// once for the composer. An edit landing between the two shipped an artifact the validator never saw.
/// The deploy now validates and composes ONE snapshot, so an edit that lands mid-deploy is either
/// validated (it was in the snapshot) or absent from the artifact (it was not) — never shipped unseen.
/// </summary>
public sealed class AdminOperationsActorSingleSnapshotTests : ControlPlaneActorTestBase
{
private const string LateEquipmentId = "EQ-late-and-NOT-canonical";
/// <summary>
/// An Equipment with a non-canonical id (the validator rejects those with <c>EquipmentIdNotDerived</c>)
/// is committed by another context while the snapshot read is in progress, AFTER the Equipment table
/// has been read. The validator cannot see it, so the deploy is Accepted — and the artifact must not
/// carry it either. Positive control: the row really landed in the DB, and a deploy started after it
/// is Rejected on exactly that rule.
/// </summary>
[Fact]
public void Edit_landing_between_validate_and_compose_cannot_ship_unvalidated()
{
// One shared root: an InMemory store is scoped to the internal service provider, and a context
// carrying an interceptor builds its own provider — without the shared root the intercepting
// context would read an EMPTY database and the whole test would be vacuous.
var dbName = Guid.NewGuid().ToString("N");
var root = new InMemoryDatabaseRoot();
var plain = new RootedDbFactory(dbName, root, interceptor: null);
using (var db = plain.CreateDbContext())
{
// A Tag to materialise — Tags is read AFTER Equipment by the snapshot factory, so an insert
// fired on the first Tag lands "between" the two tables of one read.
db.DriverInstances.Add(new DriverInstance
{
DriverInstanceId = "d", ClusterId = "", Name = "drv", DriverType = "Modbus", DriverConfig = "{}",
});
db.Devices.Add(new Device { DeviceId = "dev", DriverInstanceId = "d", Name = "dev", DeviceConfig = "{}" });
db.Tags.Add(new Tag
{
TagId = "tag-speed", DeviceId = "dev", Name = "speed", DataType = "Float",
AccessLevel = TagAccessLevel.Read, TagConfig = "{}",
});
db.SaveChanges();
}
var injecting = new RootedDbFactory(dbName, root, new LateEquipmentInterceptor(plain));
var coordinator = CreateTestProbe("coord");
var actor = Sys.ActorOf(AdminOperationsActor.Props(injecting, coordinator.Ref, Enumerable.Empty<IDriverProbe>()));
actor.Tell(new StartDeployment("joe", CorrelationId.NewId()));
var reply = ExpectMsg<StartDeploymentResult>(PresenceBudget);
reply.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, reply.Message);
coordinator.ExpectMsg<DispatchDeployment>(PresenceBudget);
using (var verify = plain.CreateDbContext())
{
// The injection happened...
verify.Equipment.Any(e => e.EquipmentId == LateEquipmentId).ShouldBeTrue();
// ...and the shipped artifact does not contain what the validator never saw.
var row = verify.Deployments.Single();
Inflate(row.ArtifactBlob).ShouldNotContain(LateEquipmentId);
// The coordinator is a probe here, so seal the row by hand to let the next deploy through.
row.Status = DeploymentStatus.Sealed;
verify.SaveChanges();
}
// Positive control: now that the row is visible to a fresh read, the gate rejects it.
actor.Tell(new StartDeployment("joe", CorrelationId.NewId()));
var second = ExpectMsg<StartDeploymentResult>(PresenceBudget);
second.Outcome.ShouldBe(StartDeploymentOutcome.Rejected);
second.Message.ShouldNotBeNull();
second.Message.ShouldContain("EquipmentIdNotDerived");
}
private static string Inflate(byte[] blob)
{
using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress);
using var reader = new StreamReader(gz, Encoding.UTF8);
return reader.ReadToEnd();
}
/// <summary>On the first Tag materialised, commits a non-canonical Equipment through ANOTHER context.</summary>
private sealed class LateEquipmentInterceptor(IDbContextFactory<OtOpcUaConfigDbContext> other)
: IMaterializationInterceptor
{
private int _fired;
public object InitializedInstance(MaterializationInterceptionData materializationData, object entity)
{
if (entity is Tag && Interlocked.Exchange(ref _fired, 1) == 0)
{
using var db = other.CreateDbContext();
db.Equipment.Add(new Equipment
{
EquipmentUuid = Guid.NewGuid(), EquipmentId = LateEquipmentId, Name = "late",
UnsLineId = "line-a", MachineCode = "m",
});
db.SaveChanges();
}
return entity;
}
}
private sealed class RootedDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor? interceptor)
: IDbContextFactory<OtOpcUaConfigDbContext>
{
public OtOpcUaConfigDbContext CreateDbContext()
{
var builder = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseInMemoryDatabase(dbName, root);
if (interceptor is not null) builder.AddInterceptors(interceptor);
return new OtOpcUaConfigDbContext(builder.Options);
}
}
}
@@ -0,0 +1,191 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.EntityFrameworkCore.Update;
using Shouldly;
using Xunit;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// R03:2.6 — one constraint hit used to drop the whole flush. The writer now detaches the rejected
/// row(s) and re-saves, so a duplicate <c>EventId</c> costs exactly that row.
/// </summary>
public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase
{
[Fact]
public void A_rejected_row_is_dropped_and_the_rest_of_the_batch_is_persisted()
{
var dbName = Guid.NewGuid().ToString("N");
var root = new InMemoryDatabaseRoot();
var duplicate = Guid.NewGuid();
var rejecting = new RejectingDbFactory(dbName, root, new AlwaysRejectBatchContaining(duplicate));
var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting));
var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList();
ids.Insert(7, duplicate);
foreach (var id in ids) actor.Tell(NewEvent(id));
Watch(actor);
actor.Tell(PoisonPill.Instance); // PostStop flushes
ExpectTerminated(actor);
using var db = rejecting.CreateDbContext();
var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList();
persisted.Count.ShouldBe(20, "every row but the rejected one lands");
persisted.ShouldNotContain(duplicate);
// Positive control: the rows on either side of the rejected one are present.
persisted.ShouldContain(ids[6]);
persisted.ShouldContain(ids[8]);
}
/// <summary>
/// Fix round 1 / I4 — a FAITHFUL SQL Server shape: the provider sends the batch as one statement batch,
/// so the <see cref="DbUpdateException"/> for one duplicate names EVERY entry in it. Detaching
/// <c>ex.Entries</c> therefore dropped the whole flush. The writer must drop only the rows whose
/// EventId is already persisted and keep the rest.
/// </summary>
[Fact]
public void A_whole_batch_rejection_drops_only_the_rows_already_persisted()
{
var dbName = Guid.NewGuid().ToString("N");
var root = new InMemoryDatabaseRoot();
var factory = new RejectingDbFactory(dbName, root, new RejectWholeBatchOnDuplicate());
var duplicate = Guid.NewGuid();
using (var seed = factory.CreateDbContext())
{
seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate });
seed.SaveChanges();
}
var actor = Sys.ActorOf(AuditWriterActor.Props(factory));
var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList();
ids.Insert(7, duplicate);
foreach (var id in ids) actor.Tell(NewEvent(id));
Watch(actor);
actor.Tell(PoisonPill.Instance);
ExpectTerminated(actor);
using var db = factory.CreateDbContext();
var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList();
persisted.Count.ShouldBe(21, "the 20 new rows land beside the one that was already there");
persisted.Count(id => id == duplicate).ShouldBe(1);
db.ConfigAuditLogs.Single(r => r.EventId == duplicate).Principal.ShouldBe("earlier");
}
/// <summary>The same contract against the REAL provider and the real filtered unique index.</summary>
[Fact]
[Trait("Category", "SchemaCompliance")]
public void On_sql_server_a_duplicate_event_id_costs_only_that_row()
{
using var sql = new SqlServerTestDatabase();
var duplicate = Guid.NewGuid();
using (var seed = sql.NewContext())
{
seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate });
seed.SaveChanges();
}
var actor = Sys.ActorOf(AuditWriterActor.Props(sql.Factory()));
var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList();
ids.Insert(7, duplicate);
foreach (var id in ids) actor.Tell(NewEvent(id));
Watch(actor);
actor.Tell(PoisonPill.Instance);
ExpectTerminated(actor, PresenceBudget);
using var db = sql.NewContext();
var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList();
persisted.Count.ShouldBe(21);
persisted.Count(id => id == duplicate).ShouldBe(1);
foreach (var id in ids) persisted.ShouldContain(id);
}
/// <summary>A failure that names no entry is a whole-batch failure, exactly as before.</summary>
[Fact]
public void A_connection_level_failure_still_drops_the_batch_without_looping()
{
var dbName = Guid.NewGuid().ToString("N");
var root = new InMemoryDatabaseRoot();
var rejecting = new RejectingDbFactory(dbName, root, new AlwaysFailWithoutEntries());
var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting));
for (var i = 0; i < 5; i++) actor.Tell(NewEvent(Guid.NewGuid()));
Watch(actor);
actor.Tell(PoisonPill.Instance);
ExpectTerminated(actor); // terminates ⇒ the retry loop did not spin
using var db = rejecting.CreateDbContext();
db.ConfigAuditLogs.Count().ShouldBe(0);
}
private static AuditEvent NewEvent(Guid eventId) => new()
{
EventId = eventId, Category = "Config", Action = "Edit", Actor = "joe",
OccurredAtUtc = DateTimeOffset.UtcNow, Outcome = AuditOutcome.Success,
DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(),
};
/// <summary>
/// A row the database refuses on its own terms (not a duplicate — e.g. a per-row constraint), every
/// time it is in the save. Faithful to SQL Server's batch shape: the exception names EVERY added
/// entry, so only a row-by-row retry can isolate the offender.
/// </summary>
private sealed class AlwaysRejectBatchContaining(Guid eventId) : ISaveChangesInterceptor
{
public InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result)
{
if (eventData.Context is null) return result;
var added = eventData.Context.ChangeTracker.Entries<ConfigAuditLog>()
.Where(e => e.State == EntityState.Added).ToList();
if (!added.Any(e => e.Entity.EventId == eventId)) return result;
#pragma warning disable EF1001
var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList();
#pragma warning restore EF1001
throw new DbUpdateException("The statement conflicted with a CHECK constraint (simulated, whole batch)", entries);
}
}
/// <summary>
/// Models SQL Server's batched insert: when ANY added row's EventId is already stored, the whole save
/// fails with an exception naming EVERY added entry (the InMemory provider enforces no unique index,
/// so this interceptor is the constraint).
/// </summary>
private sealed class RejectWholeBatchOnDuplicate : ISaveChangesInterceptor
{
public InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result)
{
if (eventData.Context is not OtOpcUaConfigDbContext ctx) return result;
var added = ctx.ChangeTracker.Entries<ConfigAuditLog>().Where(e => e.State == EntityState.Added).ToList();
var ids = added.Select(e => e.Entity.EventId).Where(id => id != null).ToList();
if (ids.Count == 0 || !ctx.ConfigAuditLogs.AsNoTracking().Any(r => ids.Contains(r.EventId))) return result;
#pragma warning disable EF1001
var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList();
#pragma warning restore EF1001
throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601, whole batch)", entries);
}
}
private sealed class AlwaysFailWithoutEntries : ISaveChangesInterceptor
{
public InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result) =>
throw new DbUpdateException("connection lost (simulated)");
}
private sealed class RejectingDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor interceptor)
: IDbContextFactory<OtOpcUaConfigDbContext>
{
public OtOpcUaConfigDbContext CreateDbContext() => new(
new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
.UseInMemoryDatabase(dbName, root)
.AddInterceptors(interceptor)
.Options);
}
}
@@ -930,6 +930,34 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
/// <summary>A spawned gRPC-mode actor plus the two seams the tests observe it through.</summary>
private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk);
/// <summary>
/// R03:1.3 — every contact refresh hands the dialer the CURRENT endpoint set so it can release the
/// channels of endpoints that left it.
/// </summary>
[Fact]
public void Each_contact_refresh_tells_the_dialer_which_endpoints_remain()
{
var factory = NewInMemoryDbFactory();
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false), ("host-b:4053", "host-b", 4056, true, false));
var rig = SpawnGrpcRig(factory);
AwaitContactsLoaded(rig);
// The dialer is resolved lazily on first use; a dispatch forces that, and the NEXT refresh prunes.
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5));
using (var db = factory.CreateDbContext())
{
db.ClusterNodes.Single(n => n.NodeId == "host-b:4053").MaintenanceMode = true;
db.SaveChanges();
}
rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts());
AwaitCondition(
() => rig.Dialer.RetainCalls.Any(set => set.Count == 1 && set.Single().Contains("host-a")),
TimeSpan.FromSeconds(5));
rig.Dialer.RetainCalls.Last().ShouldBe(new[] { "http://host-a:4056" });
}
private GrpcRig SpawnGrpcRig(
IDbContextFactory<OtOpcUaConfigDbContext> factory,
RecordingDialer? dialer = null,
@@ -1014,6 +1042,19 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
: Task.CompletedTask;
}
/// <summary>Every endpoint set handed to <see cref="RetainOnly"/>, in order.</summary>
public IReadOnlyList<IReadOnlyCollection<string>> RetainCalls
{
get { lock (_gate) return _retainCalls.ToList(); }
}
private readonly List<IReadOnlyCollection<string>> _retainCalls = [];
public void RetainOnly(IReadOnlyCollection<string> nodeEndpoints)
{
lock (_gate) _retainCalls.Add(nodeEndpoints.ToList());
}
/// <summary>One recorded dial.</summary>
public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId);
}
@@ -224,6 +224,35 @@ public sealed class MeshGrpcCommandDialerTests
Should.NotThrow(dialer.Dispose);
}
/// <summary>
/// R03:1.3 — a channel for an endpoint that left the dispatch set (host renamed, port moved) is
/// released on the next refresh instead of living, with its idle connection, for the life of central.
/// Positive control: a dial to that endpoint afterwards rebuilds a channel.
/// </summary>
[Fact]
public async Task RetainOnly_releases_channels_for_endpoints_that_left_the_dispatch_set()
{
var dialer = new MeshGrpcCommandDialer(
new FakePskProvider(("SITE-A", "psk-site-a")),
Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }),
NullLogger<MeshGrpcCommandDialer>.Instance);
foreach (var endpoint in new[] { "http://127.0.0.1:1", "http://127.0.0.1:2" })
{
await Should.ThrowAsync<RpcException>(
() => dialer.DispatchAsync(endpoint, Envelope(), "SITE-A", CancellationToken.None));
}
dialer.CachedChannelCount.ShouldBe(2);
dialer.RetainOnly(new[] { "http://127.0.0.1:1" });
dialer.CachedChannelCount.ShouldBe(1);
await Should.ThrowAsync<RpcException>(
() => dialer.DispatchAsync("http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None));
dialer.CachedChannelCount.ShouldBe(2, "a returning endpoint gets a fresh channel");
Should.NotThrow(dialer.Dispose);
}
[Fact]
public void It_activates_from_the_container_despite_the_optional_client_factory_parameter()
{
@@ -0,0 +1,200 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.Secrets.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
/// <summary>
/// R03:3.4 — central invalidated every cached key on every contact refresh (default 60 s), so the
/// first inbound ack per cluster per minute blocked a gRPC thread on a synchronous, 3 s-bounded store
/// read and was denied past the bound. <see cref="MeshPskProvider.RefreshAsync"/> pre-resolves the
/// keys off the request path and swaps them in atomically.
/// </summary>
public sealed class MeshPskProviderRefreshTests
{
private static MeshPskProvider Provider(ISecretResolver resolver) =>
new(Options.Create(new MeshTransportOptions()), resolver, NullLogger<MeshPskProvider>.Instance);
[Fact]
public async Task After_a_refresh_the_lookup_is_served_without_touching_the_store()
{
var resolver = new GatedResolver { Value = "k-1" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
resolver.Calls.ShouldBe(1);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("k-1");
resolver.Calls.ShouldBe(1, "the request path did not consult the store");
provider.Matches("SITE-A", "k-1").ShouldBeTrue();
}
/// <summary>
/// While a refresh is in flight the PREVIOUS key is still served (no invalidate-then-reread window);
/// once it completes the rotated key is served. Positive control on both sides of the swap.
/// </summary>
[Fact]
public async Task A_rotated_key_is_swapped_in_atomically_with_the_old_key_served_until_then()
{
var resolver = new GatedResolver { Value = "old" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.TryGetPsk("SITE-A", out var before).ShouldBeTrue();
before.ShouldBe("old");
resolver.Value = "new";
resolver.Hold();
var refresh = provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
resolver.Entered.Wait(TimeSpan.FromSeconds(5)).ShouldBeTrue();
// Mid-refresh: the old key, from memory — the store is NOT re-read on this path.
var callsMidRefresh = resolver.Calls;
provider.TryGetPsk("SITE-A", out var during).ShouldBeTrue();
during.ShouldBe("old");
resolver.Calls.ShouldBe(callsMidRefresh);
resolver.Release();
await refresh;
provider.TryGetPsk("SITE-A", out var after).ShouldBeTrue();
after.ShouldBe("new");
}
[Fact]
public async Task A_cluster_whose_refresh_fails_keeps_its_previous_key()
{
var resolver = new GatedResolver { Value = "k-1" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
resolver.Throw = new InvalidOperationException("store offline");
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("k-1");
}
/// <summary>
/// Fix round 1 / I8 — "the store says there is no such secret" is a REVOCATION, not a read failure:
/// the previous key must stop being served (from the refreshed snapshot AND the lazy cache). A thrown
/// read keeps the previous key (the test above); a definitive not-found drops it.
/// </summary>
[Fact]
public async Task A_secret_deleted_from_the_store_revokes_the_key_on_the_next_refresh()
{
var resolver = new GatedResolver { Value = "k-1" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.Matches("SITE-A", "k-1").ShouldBeTrue("positive control: the key was in service");
resolver.Value = null; // the secret was deleted
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
provider.Matches("SITE-A", "k-1").ShouldBeFalse();
}
/// <summary>The same revocation reaches a key that was first served from the LAZY cache, not a refresh.</summary>
[Fact]
public async Task A_deleted_secret_also_evicts_a_lazily_cached_key()
{
var resolver = new GatedResolver { Value = "k-lazy" };
var provider = Provider(resolver);
provider.TryGetPsk("SITE-A", out var lazy).ShouldBeTrue();
lazy.ShouldBe("k-lazy");
resolver.Value = null;
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.TryGetPsk("SITE-A", out _).ShouldBeFalse();
}
/// <summary>Fix round 1 minor — the refresh fans out to the store with BOUNDED concurrency, not one read per cluster at once.</summary>
[Fact]
public async Task Refresh_store_reads_are_concurrency_bounded()
{
var resolver = new ConcurrencyTrackingResolver();
var provider = Provider(resolver);
var clusters = Enumerable.Range(0, 40).Select(i => $"SITE-{i}").ToList();
await provider.RefreshAsync(clusters, CancellationToken.None);
resolver.Calls.ShouldBe(40);
resolver.MaxConcurrent.ShouldBeLessThanOrEqualTo(MeshPskProvider.MaxConcurrentRefreshReads);
resolver.MaxConcurrent.ShouldBeGreaterThan(1, "still concurrent, just bounded");
foreach (var c in clusters) provider.TryGetPsk(c, out _).ShouldBeTrue();
}
private sealed class ConcurrencyTrackingResolver : ISecretResolver
{
private int _inFlight;
private int _max;
private int _calls;
public int MaxConcurrent => Volatile.Read(ref _max);
public int Calls => Volatile.Read(ref _calls);
public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
{
Interlocked.Increment(ref _calls);
var now = Interlocked.Increment(ref _inFlight);
int seen;
while (now > (seen = Volatile.Read(ref _max)) && Interlocked.CompareExchange(ref _max, now, seen) != seen) { }
await Task.Delay(20, ct);
Interlocked.Decrement(ref _inFlight);
return "k-" + name.Value;
}
}
/// <summary>A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour).</summary>
[Fact]
public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily()
{
var resolver = new GatedResolver { Value = "k-b" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.TryGetPsk("SITE-B", out var psk).ShouldBeTrue();
psk.ShouldBe("k-b");
resolver.Calls.ShouldBe(2);
}
[Fact]
public async Task InvalidateAll_drops_refreshed_keys_too()
{
var resolver = new GatedResolver { Value = "k-1" };
var provider = Provider(resolver);
await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None);
provider.InvalidateAll();
resolver.Value = "k-2";
provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue();
psk.ShouldBe("k-2");
}
private sealed class GatedResolver : ISecretResolver
{
private int _calls;
private volatile TaskCompletionSource? _gate;
public string? Value { get; set; }
public Exception? Throw { get; set; }
public int Calls => Volatile.Read(ref _calls);
public ManualResetEventSlim Entered { get; } = new();
public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
public void Release() => _gate?.TrySetResult();
public async Task<string?> GetAsync(SecretName name, CancellationToken ct)
{
Interlocked.Increment(ref _calls);
Entered.Set();
if (_gate is { } gate) await gate.Task.WaitAsync(ct);
if (Throw is not null) throw Throw;
return Value;
}
}
}
@@ -0,0 +1,87 @@
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text.Json;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Validation;
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// Fix round 1 minor — revision-hash stability across the R03:3.2 refactor. The pre-change composer read
/// every table with a SQL <c>ORDER BY</c> (the database's case-INSENSITIVE collation); the refactor re-sorted
/// in memory with an ORDINAL comparer, which orders mixed-case keys differently ("B-1" before "a-2"). The
/// same configuration would then hash differently after the upgrade: drift detection reports every node
/// drifted, and a re-deploy of an unchanged config shows a "new" revision. This pins the composer's output
/// BYTE-identical to a verbatim replica of the pre-change composer on the real provider.
/// </summary>
[Trait("Category", "SchemaCompliance")]
public sealed class ConfigComposerSqlServerHashStabilityTests
{
[Fact]
public async Task Composer_output_is_byte_identical_to_the_pre_change_composer_for_mixed_case_keys()
{
using var sql = new SqlServerTestDatabase();
await using (var seed = sql.NewContext())
{
// Collation order a-2, B-1, c-3; ordinal order B-1, a-2, c-3.
foreach (var id in new[] { "c-3", "B-1", "a-2" })
seed.Scripts.Add(new Script { ScriptId = id, Name = id, SourceCode = "return 1;", SourceHash = id });
await seed.SaveChangesAsync();
}
await using var legacyDb = sql.NewContext();
var legacy = await LegacySnapshotAndFlattenAsync(legacyDb);
await using var viaDb = sql.NewContext();
var current = await ConfigComposer.SnapshotAndFlattenAsync(viaDb);
current.Blob.ShouldBe(legacy.Blob);
current.RevisionHash.ShouldBe(legacy.RevisionHash);
// The deploy path's form (validate + compose from ONE snapshot) must agree too.
await using var deployDb = sql.NewContext();
var viaSnapshot = ConfigComposer.Compose(await DraftSnapshotFactory.FromConfigDbAsync(deployDb));
viaSnapshot.RevisionHash.ShouldBe(legacy.RevisionHash);
}
/// <summary>Verbatim replica of <c>ConfigComposer.SnapshotAndFlattenAsync</c> before R03:3.2 (ad2b5c40).</summary>
private static async Task<(byte[] Blob, string RevisionHash)> LegacySnapshotAndFlattenAsync(OtOpcUaConfigDbContext db)
{
var ct = CancellationToken.None;
var snapshot = new
{
Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct),
Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct),
RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct),
DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct),
Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct),
TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct),
Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct),
Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct),
UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct),
PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct),
UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct),
UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct),
NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct),
ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct),
Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct),
VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct),
ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct),
};
var json = JsonSerializer.SerializeToUtf8Bytes(snapshot, new JsonSerializerOptions
{
WriteIndented = false,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never,
});
using var output = new MemoryStream(json.Length / 8);
using (var gzip = new GZipStream(output, CompressionLevel.Fastest, leaveOpen: true))
gzip.Write(json, 0, json.Length);
var blob = output.ToArray();
return (blob, Convert.ToHexStringLower(SHA256.HashData(blob)));
}
}
@@ -222,6 +222,47 @@ public sealed class ConfigComposerTests : ControlPlaneActorTestBase
withoutOverride.ResilienceConfig.ShouldBeNull("an unset override stays null ⇒ tier defaults at spawn");
}
/// <summary>
/// R03:3.2 — the deploy path composes from the <see cref="Configuration.Validation.DraftSnapshot"/> it
/// validated rather than re-reading the DB. <see cref="ConfigComposer.Compose"/> over the factory's
/// snapshot must therefore be byte-identical to the DB overload, or the drift probe (which still
/// uses the DB overload) would report every deployment as drifted.
/// </summary>
[Fact]
public async Task Compose_over_the_factory_snapshot_is_byte_identical_to_the_db_overload()
{
var f = NewInMemoryDbFactory();
await using (var db = f.CreateDbContext())
{
db.ServerClusters.Add(NewCluster("cluster-b"));
db.ServerClusters.Add(NewCluster("cluster-a"));
db.ApiKeys.Add(new ApiKey
{
ApiKeyId = "k1", Name = "scada", SecretHash = "h", Principal = "scadabridge", CreatedUtc = FixedTimestamp,
});
db.Scripts.Add(new Script { ScriptId = "s-1", Name = "s", SourceCode = "return 1;", SourceHash = "x" });
await db.SaveChangesAsync();
}
var viaDb = await ConfigComposer.SnapshotAndFlattenAsync(f.CreateDbContext());
var snapshot = await Configuration.Validation.DraftSnapshotFactory.FromConfigDbAsync(f.CreateDbContext());
var viaSnapshot = ConfigComposer.Compose(snapshot);
viaSnapshot.RevisionHash.ShouldBe(viaDb.RevisionHash);
viaSnapshot.Blob.ShouldBe(viaDb.Blob);
// The composed content is the snapshot's, not something re-read: the keys and clusters are in it.
var json = Inflate(viaSnapshot.Blob);
json.ShouldContain("\"ApiKeyId\":\"k1\"");
json.IndexOf("cluster-a", StringComparison.Ordinal).ShouldBeLessThan(json.IndexOf("cluster-b", StringComparison.Ordinal));
}
private static string Inflate(byte[] blob)
{
using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress);
using var reader = new StreamReader(gz, Encoding.UTF8);
return reader.ReadToEnd();
}
private static readonly DateTime FixedTimestamp = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
private static ServerCluster NewCluster(string id) => new()
@@ -0,0 +1,440 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// Review 03 §2.2: a <c>Deployment</c> row left in <c>Dispatching</c> with NO <c>NodeDeploymentState</c>
/// rows — <c>HandleDispatch</c> threw before its <c>SaveChanges</c>, or the process died between the
/// AdminOps row write and the coordinator's <c>Tell</c> — used to be recovered with an EMPTY expected
/// set, at which point the deadline hit the <c>0 == 0</c> "race" branch and returned without touching
/// the row. Every later <c>StartDeployment</c> was then refused <c>AnotherDeploymentInFlight</c> forever.
/// </summary>
public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase
{
private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64));
private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);
/// <summary>
/// A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it,
/// the expected set is seeded from the enabled rows, and the node's ack seals it.
/// </summary>
[Fact]
public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals()
{
var dbFactory = NewInMemoryDbFactory();
var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
SeedNodes(dbFactory, "site-a-1:4053");
// No DispatchDeployment is ever sent — PreStart alone has to recover it.
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" });
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget);
actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget);
}
/// <summary>
/// A lost dispatch that is already past the apply deadline is closed as <c>TimedOut</c> with a reason
/// naming the condition — never left in <c>Dispatching</c> — and the next deployment proceeds
/// (positive control: it is dispatched, acked and sealed through the same coordinator).
/// </summary>
[Fact]
public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged()
{
var dbFactory = NewInMemoryDbFactory();
var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10));
SeedNodes(dbFactory, "site-a-1:4053");
var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value);
row.Status.ShouldBe(DeploymentStatus.TimedOut);
row.FailureReason.ShouldNotBeNull();
row.FailureReason.ShouldContain("lost");
}, duration: PresenceBudget);
// Positive control: the coordinator is not wedged — a fresh deployment runs end to end.
var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget);
actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget);
}
/// <summary>
/// An <c>AwaitingApplyAcks</c> row with no rows (the process died between the status flip and the
/// empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh
/// dispatch would — the recovery keys on the missing expected set, not on the status value.
/// </summary>
[Fact]
public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty()
{
var dbFactory = NewInMemoryDbFactory();
SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow);
Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
db.NodeDeploymentStates.ShouldBeEmpty();
}, duration: PresenceBudget);
}
/// <summary>
/// The brief's scenario end to end: <c>HandleDispatch</c> throws once (a transient fault seeding the
/// expected set). The coordinator restarts, recovers the <c>Dispatching</c> row, re-dispatches it and
/// the deployment completes; a second <c>StartDeployment</c> is then Accepted (positive control: a
/// normal deploy seals).
/// </summary>
[Fact]
public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds()
{
var dbName = Guid.NewGuid().ToString("N");
var adminFactory = NewInMemoryDbFactory(dbName);
var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName));
SeedNodes(adminFactory, "site-a-1:4053");
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout));
var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty<IDriverProbe>()));
// Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and
// the actor would be stopped, not restarted). Identify is processed after creation, so a
// resolved selection proves PreStart completed.
await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget);
faulting.FailNext();
admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
var first = ExpectMsg<StartDeploymentResult>(PresenceBudget);
first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
first.DeploymentId.ShouldNotBeNull();
// The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands.
AwaitAssert(() =>
{
faulting.Faulted.ShouldBeTrue();
using var db = adminFactory.CreateDbContext();
db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1);
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget);
coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = adminFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget);
// Positive control: nothing is wedged — the next deploy is Accepted and seals too.
admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
var second = ExpectMsg<StartDeploymentResult>(PresenceBudget);
second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = adminFactory.CreateDbContext();
db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2);
}, duration: PresenceBudget);
}
/// <summary>
/// Fix round 1 / I7 — the window PreStart cannot see: the Dispatching row is committed AFTER the
/// coordinator's PreStart ran, and the coordinator is never told (the admin singleton died between
/// its commit and the Tell). Nothing would ever close it. The in-flight check now closes a
/// Dispatching row with NO expected-ack rows once it is past the apply deadline, so the next
/// StartDeployment is Accepted (and seals — positive control on the pipeline).
/// </summary>
[Fact]
public async Task Lost_dispatch_committed_after_PreStart_is_closed_by_the_next_StartDeployment()
{
var dbFactory = NewInMemoryDbFactory();
SeedNodes(dbFactory, "site-a-1:4053");
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart saw nothing
var stale = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10));
var admin = Sys.ActorOf(AdminOperationsActor.Props(
dbFactory, coordinator, Enumerable.Empty<IDriverProbe>(), applyDeadline: NoTimeout));
admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
var result = ExpectMsg<StartDeploymentResult>(PresenceBudget);
result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, result.Message);
using (var db = dbFactory.CreateDbContext())
{
var row = db.Deployments.Single(d => d.DeploymentId == stale.Value);
row.Status.ShouldBe(DeploymentStatus.TimedOut);
row.FailureReason.ShouldNotBeNull();
row.FailureReason.ShouldContain("lost");
}
coordinator.Tell(new ApplyAck(result.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"),
ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single(d => d.DeploymentId == result.DeploymentId.Value.Value).Status
.ShouldBe(DeploymentStatus.Sealed);
}, duration: PresenceBudget);
}
/// <summary>Control for the closure: the same row INSIDE the deadline is still in flight and still refuses.</summary>
[Fact]
public void Dispatching_row_inside_the_deadline_still_refuses_a_new_deployment()
{
var dbFactory = NewInMemoryDbFactory();
var coordinator = CreateTestProbe();
var pending = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(-5));
var admin = Sys.ActorOf(AdminOperationsActor.Props(
dbFactory, coordinator.Ref, Enumerable.Empty<IDriverProbe>(), applyDeadline: NoTimeout));
admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
var result = ExpectMsg<StartDeploymentResult>(PresenceBudget);
result.Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight);
result.DeploymentId.ShouldBe(pending);
using var db = dbFactory.CreateDbContext();
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching);
}
/// <summary>
/// Fix round 1 minor — a late or duplicated <c>DispatchDeployment</c> for a row that is already
/// terminal (it raced the lost-dispatch closure above, or a re-delivery after a seal) must not flip
/// it back to AwaitingApplyAcks or seed an expected set. Positive control: a live row dispatched right
/// after it IS seeded (so the first message was processed, not merely not-yet-processed).
/// </summary>
[Theory]
[InlineData(DeploymentStatus.Sealed)]
[InlineData(DeploymentStatus.TimedOut)]
[InlineData(DeploymentStatus.PartiallyFailed)]
public void Dispatch_of_a_terminal_row_is_ignored(DeploymentStatus terminal)
{
var dbFactory = NewInMemoryDbFactory();
SeedNodes(dbFactory, "site-a-1:4053");
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));
var done = SeedDeployment(dbFactory, terminal, DateTime.UtcNow.AddMinutes(-1));
coordinator.Tell(new DispatchDeployment(done, TestRevision, CorrelationId.NewId()));
var live = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = dbFactory.CreateDbContext();
db.Deployments.Single(d => d.DeploymentId == live.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget);
using var verify = dbFactory.CreateDbContext();
verify.Deployments.Single(d => d.DeploymentId == done.Value).Status.ShouldBe(terminal);
verify.NodeDeploymentStates.Count(s => s.DeploymentId == done.Value).ShouldBe(0);
}
/// <summary>
/// Fix round 1 minor — status transitions update a blob-free stub (key + RowVersion) instead of
/// materialising the multi-MB row. Proven on the REAL provider, where the stub's RowVersion is the
/// optimistic-concurrency predicate: dispatch → ack → Sealed lands, and the artifact bytes are
/// untouched by the partial UPDATEs.
/// </summary>
[Fact]
[Trait("Category", "SchemaCompliance")]
public void On_sql_server_status_transitions_do_not_touch_the_artifact_blob()
{
using var sql = new SqlServerTestDatabase();
var factory = sql.Factory();
SeedNodes(factory, "site-a-1:4053");
var blob = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray();
var id = DeploymentId.NewId();
using (var db = sql.NewContext())
{
db.Deployments.Add(new Deployment
{
DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching,
CreatedBy = "test", ArtifactBlob = blob,
});
db.SaveChanges();
}
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(factory, NoTimeout));
AwaitAssert(() =>
{
using var db = sql.NewContext();
db.Deployments.AsNoTracking().Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget); // PreStart re-drives the un-dispatched row
coordinator.Tell(new ApplyAck(id, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = sql.NewContext();
var row = db.Deployments.AsNoTracking().Single();
row.Status.ShouldBe(DeploymentStatus.Sealed);
row.SealedAtUtc.ShouldNotBeNull();
row.ArtifactBlob.ShouldBe(blob);
}, duration: PresenceBudget);
}
/// <summary>
/// Re-review 1 — the reopen race. AdminOps' lost-dispatch closure commits TimedOut AFTER the coordinator's
/// terminal guard read the row as Dispatching but BEFORE the AwaitingApplyAcks flip. The flip is
/// conditioned on the guard's RowVersion, so on SQL Server it becomes a concurrency conflict and the row
/// stays TimedOut with no expected set. Positive control: the coordinator is not wedged — the next live
/// dispatch is seeded and flipped.
/// </summary>
[Fact]
[Trait("Category", "SchemaCompliance")]
public async Task On_sql_server_a_closure_between_guard_and_flip_is_not_reopened()
{
using var sql = new SqlServerTestDatabase();
var inner = sql.Factory();
SeedNodes(inner, "site-a-1:4053");
var interposer = new InterposingDbFactory(inner);
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(interposer, NoTimeout));
await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart ran over an empty table
// The race proper: a new Dispatching row whose dispatch is interleaved with an AdminOps-style closure.
var victim = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow);
interposer.OnSecondCreate(() =>
{
using var db = sql.NewContext();
db.Database.ExecuteSqlRaw(
"UPDATE Deployment SET Status = {0}, FailureReason = N'closed as lost' WHERE DeploymentId = {1}",
(int)DeploymentStatus.TimedOut, victim.Value);
});
coordinator.Tell(new DispatchDeployment(victim, TestRevision, CorrelationId.NewId()));
AwaitAssert(() => interposer.Fired.ShouldBeTrue(), duration: PresenceBudget);
// Positive control + ordering: a live row dispatched after it IS flipped, so the victim's dispatch was processed.
var live = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow);
coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId()));
AwaitAssert(() =>
{
using var db = sql.NewContext();
db.Deployments.AsNoTracking().Single(d => d.DeploymentId == live.Value).Status
.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
}, duration: PresenceBudget);
using var verify = sql.NewContext();
verify.Deployments.AsNoTracking().Single(d => d.DeploymentId == victim.Value).Status
.ShouldBe(DeploymentStatus.TimedOut, "the closure must not be reopened");
verify.NodeDeploymentStates.Count(s => s.DeploymentId == victim.Value).ShouldBe(0);
}
/// <summary>Runs an action just before handing out the SECOND context after arming (HandleDispatch: guard, then flip).</summary>
private sealed class InterposingDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner)
: IDbContextFactory<OtOpcUaConfigDbContext>
{
private Action? _action;
private int _countdown;
public volatile bool Fired;
public void OnSecondCreate(Action action)
{
_action = action;
Volatile.Write(ref _countdown, 2);
}
public OtOpcUaConfigDbContext CreateDbContext()
{
if (Volatile.Read(ref _countdown) > 0 && Interlocked.Decrement(ref _countdown) == 0)
{
_action?.Invoke();
Fired = true;
}
return inner.CreateDbContext();
}
}
private static DeploymentId SeedDeployment(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentStatus status, DateTime createdAtUtc)
{
var id = DeploymentId.NewId();
using var db = dbFactory.CreateDbContext();
db.Deployments.Add(new Deployment
{
DeploymentId = id.Value,
RevisionHash = TestRevision.Value,
Status = status,
CreatedBy = "test",
CreatedAtUtc = createdAtUtc,
});
db.SaveChanges();
return id;
}
private static void SeedNodes(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] nodeIds)
{
using var db = dbFactory.CreateDbContext();
db.ServerClusters.Add(new ServerCluster
{
ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
});
foreach (var nodeId in nodeIds)
{
db.ClusterNodes.Add(new ClusterNode
{
NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0],
ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test",
});
}
db.SaveChanges();
}
/// <summary>Throws from exactly one <c>CreateDbContext</c> call after <see cref="FailNext"/>.</summary>
private sealed class OneShotFaultingDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner)
: IDbContextFactory<OtOpcUaConfigDbContext>
{
private int _armed;
public bool Faulted { get; private set; }
public void FailNext() => Interlocked.Exchange(ref _armed, 1);
public OtOpcUaConfigDbContext CreateDbContext()
{
if (Interlocked.Exchange(ref _armed, 0) == 1)
{
Faulted = true;
throw new InvalidOperationException("injected transient fault seeding the expected set");
}
return inner.CreateDbContext();
}
}
}
@@ -0,0 +1,233 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// R03:1.2 — every deploy stored the whole fleet snapshot forever. The retention sweep empties the
/// blob on old terminal rows (never deletes a row, never touches an in-flight one, never the newest N,
/// never one a node applied recently).
/// </summary>
public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase
{
private static readonly DateTime SweepNow = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc);
[Fact]
public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact()
{
var f = NewInMemoryDbFactory();
var ids = Seed(f, sealedCount: 12, mixInTimedOut: false); // ids[0] newest … ids[11] oldest
var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 };
int pruned;
await using (var db = f.CreateDbContext())
pruned = await DeploymentRetentionActor.SweepAsync(db, options, SweepNow);
pruned.ShouldBe(7);
await using var verify = f.CreateDbContext();
foreach (var id in ids.Take(5))
(await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.Length.ShouldBeGreaterThan(0, "newest 5 keep their blob");
foreach (var id in ids.Skip(5))
(await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.ShouldBeEmpty("older rows are emptied");
// Rows survive: the audit record is intact.
(await verify.Deployments.CountAsync()).ShouldBe(12);
// Idempotent: a second sweep finds nothing.
await using var again = f.CreateDbContext();
(await DeploymentRetentionActor.SweepAsync(again, options, SweepNow)).ShouldBe(0);
}
[Fact]
public async Task Sweep_keeps_a_row_a_node_applied_recently_and_never_touches_an_in_flight_row()
{
var f = NewInMemoryDbFactory();
var ids = Seed(f, sealedCount: 4);
var inFlight = Guid.NewGuid();
await using (var db = f.CreateDbContext())
{
db.ClusterNodes.Add(new ClusterNode
{
NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1",
ApplicationUri = "urn:x", Enabled = true, CreatedBy = "test",
});
// ids[3] is the oldest; a node applied it yesterday, so it stays.
db.NodeDeploymentStates.Add(new NodeDeploymentState
{
NodeId = "site-a-1:4053", DeploymentId = ids[3], Status = NodeDeploymentStatus.Applied,
StartedAtUtc = SweepNow.AddDays(-1), AppliedAtUtc = SweepNow.AddDays(-1),
});
// ids[2]: applied long ago — not protected.
db.NodeDeploymentStates.Add(new NodeDeploymentState
{
NodeId = "site-a-1:4053", DeploymentId = ids[2], Status = NodeDeploymentStatus.Applied,
StartedAtUtc = SweepNow.AddDays(-90), AppliedAtUtc = SweepNow.AddDays(-90),
});
// An ancient in-flight row with a blob — never touched.
db.Deployments.Add(new Deployment
{
DeploymentId = inFlight, RevisionHash = new string('f', 64), Status = DeploymentStatus.AwaitingApplyAcks,
CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-400), ArtifactBlob = new byte[] { 1, 2, 3 },
});
await db.SaveChangesAsync();
}
var options = new DeploymentRetentionOptions { KeepSealed = 1, ReferencedDays = 30 };
await using (var db = f.CreateDbContext())
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); // ids[1], ids[2]
await using var verify = f.CreateDbContext();
(await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty("newest");
(await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[1])).ArtifactBlob.ShouldBeEmpty();
(await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[2])).ArtifactBlob.ShouldBeEmpty("applied 90 days ago");
(await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[3])).ArtifactBlob.ShouldNotBeEmpty("applied yesterday");
(await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight");
}
/// <summary>
/// Fix round 1 / I3 — the keep window counts SEALED rows, not "terminal" ones. A burst of 60 failed
/// deploys used to fill a 50-row terminal window on its own and push the fleet's last good (Sealed)
/// artifact out of it — the one every boot / drift / fallback path reads. The newest Sealed blob is
/// never pruned; the unreferenced TimedOut rows are (positive control: the sweep did run).
/// </summary>
[Fact]
public async Task Sixty_timed_out_deploys_never_prune_the_newest_sealed_blob()
{
var f = NewInMemoryDbFactory();
var sealedId = Guid.NewGuid();
var timedOut = new List<Guid>();
await using (var db = f.CreateDbContext())
{
db.Deployments.Add(new Deployment
{
DeploymentId = sealedId, RevisionHash = new string('s', 64), Status = DeploymentStatus.Sealed,
CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-200), SealedAtUtc = SweepNow.AddDays(-200),
ArtifactBlob = new byte[] { 1, 2, 3 },
});
for (var i = 0; i < 60; i++)
{
var id = Guid.NewGuid();
timedOut.Add(id);
db.Deployments.Add(new Deployment
{
DeploymentId = id, RevisionHash = new string('t', 64), Status = DeploymentStatus.TimedOut,
CreatedBy = "test", CreatedAtUtc = SweepNow.AddHours(-i), ArtifactBlob = new byte[] { 9 },
});
}
await db.SaveChangesAsync();
}
var options = new DeploymentRetentionOptions { KeepSealed = 50, ReferencedDays = 30, MaxRowsPerSweep = 200 };
await using (var db = f.CreateDbContext())
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(60);
await using var verify = f.CreateDbContext();
(await verify.Deployments.SingleAsync(d => d.DeploymentId == sealedId)).ArtifactBlob
.ShouldNotBeEmpty("the newest Sealed artifact is never pruned");
(await verify.Deployments.CountAsync(d => timedOut.Contains(d.DeploymentId) && d.ArtifactBlob.Length == 0))
.ShouldBe(60);
}
/// <summary>Even with KeepSealed misconfigured to 0, the newest Sealed row keeps its blob.</summary>
[Fact]
public async Task The_newest_sealed_blob_survives_a_zero_keep_window()
{
var f = NewInMemoryDbFactory();
var ids = Seed(f, sealedCount: 3, mixInTimedOut: false);
await using (var db = f.CreateDbContext())
(await DeploymentRetentionActor.SweepAsync(db, new DeploymentRetentionOptions { KeepSealed = 0 }, SweepNow))
.ShouldBe(2);
await using var verify = f.CreateDbContext();
(await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty();
}
[Fact]
public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls()
{
var f = NewInMemoryDbFactory();
Seed(f, sealedCount: 10);
var options = new DeploymentRetentionOptions { KeepSealed = 2, MaxRowsPerSweep = 3 };
await using (var db = f.CreateDbContext())
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3);
await using (var db = f.CreateDbContext())
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3);
await using (var db = f.CreateDbContext())
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2);
await using (var db = f.CreateDbContext())
(await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(0);
}
/// <summary>The actor runs the same rule off its thread and reports back.</summary>
[Fact]
public void Actor_sweeps_on_request_and_reports_the_count()
{
var f = NewInMemoryDbFactory();
Seed(f, sealedCount: 6);
var actor = Sys.ActorOf(DeploymentRetentionActor.Props(f, new DeploymentRetentionOptions
{
KeepSealed = 4, InitialDelaySeconds = 3600, IntervalMinutes = 60,
}));
actor.Tell(DeploymentRetentionActor.Sweep.Instance);
ExpectMsg<DeploymentRetentionActor.SweepCompleted>(PresenceBudget).Pruned.ShouldBe(2);
using var verify = f.CreateDbContext();
verify.Deployments.Count(d => d.ArtifactBlob.Length == 0).ShouldBe(2);
}
[Fact]
public void Options_bind_from_configuration_with_defaults()
{
var defaults = DeploymentRetentionOptions.Bind(null);
defaults.Enabled.ShouldBeTrue();
defaults.KeepSealed.ShouldBe(50);
defaults.ReferencedDays.ShouldBe(30);
defaults.IntervalMinutes.ShouldBe(60);
var config = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["Deployments:Retention:KeepSealed"] = "7",
["Deployments:Retention:Enabled"] = "false",
})
.Build();
var bound = DeploymentRetentionOptions.Bind(config);
bound.KeepSealed.ShouldBe(7);
bound.Enabled.ShouldBeFalse();
bound.ReferencedDays.ShouldBe(30);
}
/// <summary>Seeds sealed deployments, newest first; each one hour older than the previous.</summary>
private static List<Guid> Seed(IDbContextFactory<OtOpcUaConfigDbContext> f, int sealedCount, bool mixInTimedOut = true)
{
var ids = new List<Guid>();
using var db = f.CreateDbContext();
for (var i = 0; i < sealedCount; i++)
{
var id = Guid.NewGuid();
ids.Add(id);
db.Deployments.Add(new Deployment
{
DeploymentId = id,
RevisionHash = new string('a', 64),
Status = mixInTimedOut && i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed,
CreatedBy = "test",
CreatedAtUtc = SweepNow.AddHours(-i),
SealedAtUtc = SweepNow.AddHours(-i),
ArtifactBlob = new byte[] { 0x1f, 0x8b, (byte)i },
});
}
db.SaveChanges();
return ids;
}
}
@@ -0,0 +1,77 @@
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.OtOpcUa.Configuration;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
/// <summary>
/// A migrated, throw-away config database on the central SQL Server, for the few ControlPlane behaviours
/// the InMemory provider cannot model (a SQL Server <c>DbUpdateException</c> names the WHOLE statement
/// batch; collation-ordered reads). Same server / credentials / env overrides as
/// <c>Configuration.Tests.SchemaComplianceFixture</c> (<c>OTOPCUA_CONFIG_TEST_SERVER</c>,
/// <c>OTOPCUA_CONFIG_TEST_SA_PASSWORD</c>; default the shared docker host's dev SA), and the same database
/// name shape, so that suite's 24 h orphan reaper also reclaims a database a killed run leaks here.
/// </summary>
public sealed class SqlServerTestDatabase : IDisposable
{
private const string DefaultServer = "10.100.0.35,14330";
private const string DefaultSaPassword = "OtOpcUaDev_2026!";
/// <summary>Gets the connection string of the migrated database.</summary>
public string ConnectionString { get; }
private readonly string _databaseName;
/// <summary>Creates and migrates the database.</summary>
public SqlServerTestDatabase()
{
var server = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SERVER") ?? DefaultServer;
var saPassword = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SA_PASSWORD") ?? DefaultSaPassword;
_databaseName = $"OtOpcUaConfig_Test_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
ConnectionString =
$"Server={server};Database={_databaseName};User Id=sa;Password={saPassword};TrustServerCertificate=True;Encrypt=False;";
using var ctx = NewContext();
ctx.Database.Migrate();
}
/// <summary>A new context on the database.</summary>
/// <returns>The context.</returns>
public OtOpcUaConfigDbContext NewContext() =>
new(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>().UseSqlServer(ConnectionString).Options);
/// <summary>A factory over <see cref="NewContext"/>.</summary>
/// <returns>The factory.</returns>
public IDbContextFactory<OtOpcUaConfigDbContext> Factory() => new ContextFactory(this);
/// <summary>Drops the database (best effort; a leak is reaped by the Configuration suite).</summary>
public void Dispose()
{
try
{
SqlConnection.ClearAllPools();
var master = new SqlConnectionStringBuilder(ConnectionString) { InitialCatalog = "master" }.ConnectionString;
using var conn = new SqlConnection(master);
conn.Open();
using var cmd = conn.CreateCommand();
cmd.CommandText = @"
DECLARE @sql nvarchar(max);
IF DB_ID(@name) IS NOT NULL
BEGIN
SET @sql = N'ALTER DATABASE ' + QUOTENAME(@name) + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE; ' +
N'DROP DATABASE ' + QUOTENAME(@name) + N';';
EXEC sp_executesql @sql;
END";
cmd.Parameters.Add(new SqlParameter("@name", _databaseName));
cmd.ExecuteNonQuery();
}
catch (Exception)
{
// Best effort — see the class remarks.
}
}
private sealed class ContextFactory(SqlServerTestDatabase owner) : IDbContextFactory<OtOpcUaConfigDbContext>
{
public OtOpcUaConfigDbContext CreateDbContext() => owner.NewContext();
}
}
@@ -0,0 +1,64 @@
using Microsoft.Extensions.Logging.Abstractions;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Host.Engines;
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
/// <summary>
/// <see cref="RoslynVirtualTagEvaluator"/> runs every compiled expression through a
/// <see cref="ScriptRunGuard"/> keyed by virtual-tag id: a runaway expression leaks one orphaned pool
/// thread, not one per trigger, later triggers are refused, the tag is quarantined after three consecutive timeouts (surfacing
/// as a <c>Failure</c> the actor publishes Bad for), and the quarantine lifts at the deploy boundary.
/// </summary>
public sealed class RoslynVirtualTagEvaluatorRunGuardTests
{
private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger());
private const string SpinScript =
"var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;";
private static readonly Dictionary<string, object?> NoDeps = new();
[Fact]
public void Runaway_expression_is_refused_while_its_orphan_runs_and_a_sibling_keeps_evaluating()
{
using var sut = new RoslynVirtualTagEvaluator(
NullLogger<RoslynVirtualTagEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60));
var first = sut.Evaluate("vt-loop", SpinScript, NoDeps);
first.Success.ShouldBeFalse();
first.Reason.ShouldNotBeNull();
first.Reason.ShouldContain("timed out");
// Refusals, not strikes (fix round 1, I1): the tag stays Bad but is not quarantined by them.
sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason);
sut.QuarantinedScripts.ShouldBeEmpty();
sut.RunGuard.StartedRunCount.ShouldBe(1, "one orphan, not one per trigger");
var sibling = sut.Evaluate("vt-ok", "return 42;", NoDeps);
sibling.Success.ShouldBeTrue(sibling.Reason);
sibling.Value.ShouldBe(42);
}
[Fact]
public void ResetQuarantines_lifts_the_quarantine_as_a_redeploy_does()
{
using var sut = new RoslynVirtualTagEvaluator(
NullLogger<RoslynVirtualTagEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60),
new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 });
sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix);
sut.QuarantinedScripts.ShouldNotBeEmpty();
sut.ResetQuarantines();
sut.QuarantinedScripts.ShouldBeEmpty();
var fixedRun = sut.Evaluate("vt-loop", "return 7;", NoDeps);
fixedRun.Success.ShouldBeTrue(fixedRun.Reason);
fixedRun.Value.ShouldBe(7);
}
}
@@ -489,12 +489,18 @@ public sealed class RoslynVirtualTagEvaluatorTests
// most evaluations then hit the warm cache instead of recompiling. Neither the cadence nor the
// iteration count is a correctness knob any more — the assertions below hold for any number of
// clears landing inside a single evaluation, and for any number of evaluations completed.
var evalLoops = Enumerable.Range(0, loopCount).Select(_ => Task.Run(() =>
// One virtual-tag id PER LOOP, sharing the expression. The race under test lives in the compile
// cache, which is keyed by source, so twelve ids on one expression contend for exactly the entry
// twelve copies of one id did — while twelve concurrent evaluations of ONE id is a shape production
// never produces (one VirtualTagActor per id serialises them) and exactly the shape the per-tag
// ScriptRunGuard refuses ("previous evaluation still running", then quarantine).
var evalLoops = Enumerable.Range(0, loopCount).Select(loop => Task.Run(() =>
{
var vtagId = $"vt-race-{loop}";
var done = 0;
for (var i = 0; i < MaxIterationsPerLoop && Environment.TickCount64 < stopAtMs; i++)
{
var r = sut.Evaluate("vt-race", expr, new Dictionary<string, object?> { ["a"] = i });
var r = sut.Evaluate(vtagId, expr, new Dictionary<string, object?> { ["a"] = i });
done++;
if (!r.Success)
{
@@ -0,0 +1,54 @@
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration;
/// <summary>
/// R03:2.3 — an unset or placeholder pepper must fail host start, not surface as "Invalid API key"
/// on the first activation.
/// </summary>
public sealed class ApiKeyOptionsValidatorTests
{
private static ValidateOptionsResult Validate(ApiKeyOptions options) =>
new ApiKeyOptionsValidator().Validate(Options.DefaultName, options);
[Theory]
[InlineData("")]
[InlineData(" ")]
public void Blank_pepper_fails(string pepper)
{
var result = Validate(new ApiKeyOptions { Pepper = pepper });
result.Failed.ShouldBeTrue();
result.FailureMessage.ShouldNotBeNull();
result.FailureMessage.ShouldContain("Pepper is required");
}
[Fact]
public void Short_pepper_fails()
{
var result = Validate(new ApiKeyOptions { Pepper = "x" });
result.Failed.ShouldBeTrue();
result.FailureMessage.ShouldNotBeNull();
result.FailureMessage.ShouldContain($"at least {ApiKeyOptionsValidator.MinPepperLength}");
}
[Fact]
public void Blank_username_fails()
{
var result = Validate(new ApiKeyOptions { Pepper = new string('p', 32), Username = " " });
result.Failed.ShouldBeTrue();
result.FailureMessage.ShouldNotBeNull();
result.FailureMessage.ShouldContain("Username");
}
/// <summary>Positive control: the docker-dev rig's pepper and the defaults pass unchanged.</summary>
[Fact]
public void Configured_pepper_passes()
{
Validate(new ApiKeyOptions { Pepper = "docker-dev-apikey-pepper-not-for-production" })
.Succeeded.ShouldBeTrue();
}
}
@@ -49,6 +49,41 @@ public sealed class ActiveTierRegistrationTests
ClusterPrimaryRegistration(hasAdmin).Tags.ShouldContain(ZbHealthTags.Active);
}
/// <summary>
/// R03:2.1 + fix round 1 / I6 (coordinator ruling) — the OPC UA endpoint probe's tiers by role:
/// <list type="bullet">
/// <item>driver-only: Ready + Active (it cannot serve and must not be routed to);</item>
/// <item>fused admin+driver: Ready ONLY, until the ServiceLevel calculator consumes the fact
/// (F-C1) — Traefik routes the AdminUI on <c>/health/active</c>, so a failed SDK on the active
/// central node must not take the AdminUI out of rotation;</item>
/// <item>admin-only: absent (it starts no endpoint to be degraded by).</item>
/// </list>
/// </summary>
[Theory]
[InlineData(false, true, true, true)]
[InlineData(true, true, true, false)]
[InlineData(true, false, false, false)]
public void Opcua_endpoint_probe_tiers_follow_the_node_roles(bool hasAdmin, bool hasDriver, bool ready, bool active)
{
var registrations = new ServiceCollection()
.AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: hasDriver)
.BuildServiceProvider()
.GetRequiredService<IOptions<HealthCheckServiceOptions>>()
.Value.Registrations
.Where(r => r.Name == OpcUaEndpointHealthCheck.Name)
.ToList();
if (!ready && !active)
{
registrations.ShouldBeEmpty();
return;
}
var registration = registrations.ShouldHaveSingleItem();
registration.Tags.Contains(ZbHealthTags.Ready).ShouldBe(ready);
registration.Tags.Contains(ZbHealthTags.Active).ShouldBe(active);
}
[Fact]
public void Active_tier_check_is_not_on_the_ready_tier()
{
@@ -0,0 +1,203 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa;
/// <summary>
/// R03:3.3 — the negative cache was keyed by key id, so a flood of DISTINCT well-formed ids cost one
/// SQL query per attempt under the SessionManager lock, no credential needed. Two guards:
/// the known-id set (unknown ids refused in memory) and the miss budget (a token bucket on lookups
/// that reach the store and miss, for when the set is unavailable).
/// </summary>
public sealed class BoundedApiKeyStoreKnownIdTests
{
private static readonly DateTimeOffset T0 = new(2026, 10, 22, 12, 0, 0, TimeSpan.Zero);
[Fact]
public async Task An_unknown_id_is_refused_without_reaching_the_store_and_a_known_one_still_does()
{
var inner = new ListingStore("k-real");
var clock = new FakeClock(T0);
var sut = Bounded(inner, clock, cacheSeconds: 5);
(await sut.FindAsync("k-guess-1", T0.UtcDateTime, default)).ShouldBeNull();
(await sut.FindAsync("k-guess-2", T0.UtcDateTime, default)).ShouldBeNull();
inner.FindCalls.ShouldBe(0, "unknown ids never reach the store");
inner.ListCalls.ShouldBe(1, "one set load serves every unknown id inside the TTL");
// Positive control: the real id reaches the store and is found.
(await sut.FindAsync("k-real", T0.UtcDateTime, default)).ShouldNotBeNull();
inner.FindCalls.ShouldBe(1);
}
[Fact]
public async Task The_set_is_refreshed_after_the_TTL_so_a_newly_minted_key_becomes_visible()
{
var inner = new ListingStore("k-old");
var clock = new FakeClock(T0);
var sut = Bounded(inner, clock, cacheSeconds: 5);
(await sut.FindAsync("k-new", T0.UtcDateTime, default)).ShouldBeNull();
inner.Mint("k-new");
// Still inside the TTL: the stale set refuses it (the documented window).
clock.Advance(TimeSpan.FromSeconds(3));
(await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
inner.FindCalls.ShouldBe(0);
clock.Advance(TimeSpan.FromSeconds(3));
(await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
inner.ListCalls.ShouldBe(2);
}
[Fact]
public async Task A_failed_set_load_falls_back_to_the_per_id_lookup_and_keeps_an_older_set()
{
var inner = new ListingStore("k-real") { ListThrows = true };
var clock = new FakeClock(T0);
var sut = Bounded(inner, clock, cacheSeconds: 5);
// No set yet and the load fails: the per-id path runs (old behaviour), nothing is wedged.
(await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
inner.FindCalls.ShouldBe(1);
sut.KnownKeyIds.ShouldBeNull();
inner.ListThrows = false;
clock.Advance(TimeSpan.FromSeconds(6));
(await sut.FindAsync("k-guess-2", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
sut.KnownKeyIds.ShouldNotBeNull();
inner.FindCalls.ShouldBe(1, "once the set loaded, the guess was refused in memory");
// A later failed refresh keeps the previous set in force.
inner.ListThrows = true;
clock.Advance(TimeSpan.FromSeconds(6));
(await sut.FindAsync("k-guess-3", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
inner.FindCalls.ShouldBe(1);
}
/// <summary>With no id set available, misses are rate-limited; hits are not charged; time refills.</summary>
[Fact]
public async Task The_miss_budget_bounds_store_misses_when_no_set_is_available()
{
var inner = new ListingStore("k-real") { SupportsListing = false };
var clock = new FakeClock(T0);
var sut = Bounded(inner, clock, cacheSeconds: 0, missBurst: 3, refillPerSecond: 1);
for (var i = 0; i < 3; i++)
(await sut.FindAsync($"k-guess-{i}", T0.UtcDateTime, default)).ShouldBeNull();
inner.FindCalls.ShouldBe(3);
// Budget exhausted: refused without a store read, as a throw the authenticator denies on.
await Should.ThrowAsync<InvalidOperationException>(() => sut.FindAsync("k-guess-9", T0.UtcDateTime, default));
inner.FindCalls.ShouldBe(3);
// Refill: one second restores one token.
clock.Advance(TimeSpan.FromSeconds(1));
(await sut.FindAsync("k-guess-10", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
inner.FindCalls.ShouldBe(4);
// A hit is not a miss: it reaches the store even with the bucket empty? No — it needs a token to
// be attempted, but the token is refunded, so the next attempt still has it. Prove the refund.
clock.Advance(TimeSpan.FromSeconds(1));
(await sut.FindAsync("k-real", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
(await sut.FindAsync("k-guess-11", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
inner.FindCalls.ShouldBe(6);
}
[Fact]
public async Task The_guards_are_off_when_the_cache_is_off_restoring_a_read_per_lookup()
{
var inner = new ListingStore("k-real");
var sut = Bounded(inner, new FakeClock(T0), cacheSeconds: 0, missBurst: 0);
(await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
(await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
inner.FindCalls.ShouldBe(2);
inner.ListCalls.ShouldBe(0);
}
/// <summary>
/// Fix round 1 minor — an inner store that cannot list ids switches the known-id set off for THIS
/// decorator only. It used to do so by writing <c>KnownIdSet = false</c> into the guard options it was
/// handed, a shared options object, so one store's capability leaked into every other consumer.
/// Positive control: a second decorator over a listing store on the same options instance still uses
/// the set.
/// </summary>
[Fact]
public async Task An_unlisting_store_does_not_mutate_the_shared_guard_options()
{
var guard = new ApiKeyStoreGuardOptions { MissBurst = 100, MissRefillPerSecond = 20 };
var options = Options.Create(new ApiKeyOptions
{
Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = 5,
});
var clock = new FakeClock(T0);
var unlisting = new ListingStore("k-real") { SupportsListing = false };
var first = new BoundedApiKeyStore(unlisting, options, NullLogger<BoundedApiKeyStore>.Instance, clock, guard);
(await first.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
unlisting.FindCalls.ShouldBe(1, "no set: the per-id path ran");
guard.KnownIdSet.ShouldBeTrue("the shared options object is never written");
var listing = new ListingStore("k-real");
var second = new BoundedApiKeyStore(listing, options, NullLogger<BoundedApiKeyStore>.Instance, clock, guard);
(await second.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull();
listing.FindCalls.ShouldBe(0, "the second decorator still refuses unknown ids in memory");
listing.ListCalls.ShouldBe(1);
}
private static BoundedApiKeyStore Bounded(
IApiKeyStore inner, TimeProvider clock, int cacheSeconds, int missBurst = 100, double refillPerSecond = 20) => new(
inner,
Options.Create(new ApiKeyOptions
{
Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = cacheSeconds,
}),
NullLogger<BoundedApiKeyStore>.Instance,
clock,
new ApiKeyStoreGuardOptions { MissBurst = missBurst, MissRefillPerSecond = refillPerSecond });
private sealed class ListingStore(params string[] ids) : IApiKeyStore
{
private readonly HashSet<string> _ids = ids.ToHashSet(StringComparer.Ordinal);
private int _findCalls;
private int _listCalls;
public bool SupportsListing { get; set; } = true;
public bool ListThrows { get; set; }
public int FindCalls => _findCalls;
public int ListCalls => _listCalls;
public void Mint(string id) => _ids.Add(id);
public Task<ApiKeyLookup?> FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct)
{
Interlocked.Increment(ref _findCalls);
return Task.FromResult(_ids.Contains(apiKeyId)
? new ApiKeyLookup(new ApiKey { ApiKeyId = apiKeyId, Name = "k", SecretHash = "h", Principal = "p" }, [])
: null);
}
public Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct) =>
Task.FromResult<IReadOnlyList<NodeAcl>>([]);
public Task<IReadOnlySet<string>?> ListKeyIdsAsync(CancellationToken ct)
{
if (!SupportsListing) return Task.FromResult<IReadOnlySet<string>?>(null);
Interlocked.Increment(ref _listCalls);
if (ListThrows) throw new InvalidOperationException("ConfigDb unreachable (simulated)");
return Task.FromResult<IReadOnlySet<string>?>(_ids.ToHashSet(StringComparer.Ordinal));
}
}
private sealed class FakeClock(DateTimeOffset utcNow) : TimeProvider
{
private DateTimeOffset _now = utcNow;
public override DateTimeOffset GetUtcNow() => _now;
public void Advance(TimeSpan by) => _now = _now.Add(by);
}
}
@@ -363,7 +363,10 @@ public sealed class BoundedApiKeyStoreTests
{
var (_, inner) = SetUp();
var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock);
// The miss budget (R03:3.3) is OFF here on purpose: this fact floods 1025 distinct unknown ids — the
// exact shape the budget refuses — to pin the cache CEILING, which is a separate rule.
var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock,
guard: new ApiKeyStoreGuardOptions { MissBurst = 0 });
// Seed the real key, then flood past the ceiling with ids that do not exist.
(await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
@@ -477,7 +480,8 @@ public sealed class BoundedApiKeyStoreTests
int timeoutMs,
int cacheSeconds,
TimeProvider? clock = null,
ILogger<BoundedApiKeyStore>? log = null) => new(
ILogger<BoundedApiKeyStore>? log = null,
ApiKeyStoreGuardOptions? guard = null) => new(
inner,
Options.Create(new ApiKeyOptions
{
@@ -487,7 +491,8 @@ public sealed class BoundedApiKeyStoreTests
StoreCacheSeconds = cacheSeconds,
}),
log ?? NullLogger<BoundedApiKeyStore>.Instance,
clock ?? TimeProvider.System);
clock ?? TimeProvider.System,
guard);
private static ApiKeyOpcUaUserAuthenticator Auth(IApiKeyStore store) => new(
store,
@@ -0,0 +1,213 @@
using System.Net;
using System.Net.Sockets;
using Akka.Hosting;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using ZB.MOM.WW.OtOpcUa.Host.Health;
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa;
/// <summary>
/// R03:2.1 / 1.4 — a failed OPC UA SDK start used to be one Error line: the node stayed Ready, joined
/// its mesh, and acked deploys through the Null sink while clients got connection-refused. The start
/// outcome is now a fact (<see cref="IOpcUaEndpointState"/>) the ready/active tiers read, a driver-only
/// node rethrows so the process exits, and <c>StopAsync</c> actually stops the SDK.
/// </summary>
public sealed class OtOpcUaServerHostedServiceStartFailureTests
{
private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30);
/// <summary>
/// Port already bound ⇒ the SDK cannot listen ⇒ the state is Failed, the probe is Unhealthy and the
/// real <c>/health/ready</c> endpoint answers 503 — while StartAsync itself returns (fused node).
/// Positive control: the same endpoint answers 200 once the state is Started.
/// </summary>
[Fact]
public async Task Port_in_use_marks_the_endpoint_failed_and_health_ready_answers_503()
{
using var deadline = new CancellationTokenSource(Deadline);
using var squatter = SquatAllInterfaces(out var port);
var pki = NewPki();
var state = new OpcUaEndpointState();
var fused = EffectiveRoles.Resolve(new[] { "admin", "driver" }, envRoles: null);
var service = NewService(port, pki, state, fused);
try
{
await service.StartAsync(deadline.Token); // must NOT throw on a fused node
state.Status.ShouldBe(OpcUaEndpointStatus.Failed);
state.FailureReason.ShouldNotBeNullOrWhiteSpace();
var probe = await new OpcUaEndpointHealthCheck(state).CheckHealthAsync(new HealthCheckContext());
probe.Status.ShouldBe(HealthStatus.Unhealthy);
probe.Description.ShouldNotBeNull();
probe.Description.ShouldContain("failed to start");
await using var web = BuildHealthHost(state);
await web.StartAsync(deadline.Token);
using var http = new HttpClient { BaseAddress = new Uri(web.Urls.First()) };
(await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable);
(await http.GetAsync("/health/active", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable);
// Positive control on the endpoint: the same host answers 200 for a started endpoint, so the
// 503 above measured the probe and not a broken health pipeline.
state.MarkStarted();
(await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.OK);
await web.StopAsync(deadline.Token);
}
finally
{
await service.DisposeAsync();
squatter.Stop();
Cleanup(pki);
}
}
/// <summary>A driver-only node has nothing else to stay up for: the failed start propagates.</summary>
[Fact]
public async Task Driver_only_node_rethrows_when_the_sdk_fails_to_start()
{
using var deadline = new CancellationTokenSource(Deadline);
using var squatter = SquatAllInterfaces(out var port);
var pki = NewPki();
var state = new OpcUaEndpointState();
var driverOnly = EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null);
var service = NewService(port, pki, state, driverOnly);
try
{
await Should.ThrowAsync<Exception>(() => service.StartAsync(deadline.Token));
state.Status.ShouldBe(OpcUaEndpointStatus.Failed);
}
finally
{
await service.DisposeAsync();
squatter.Stop();
Cleanup(pki);
}
}
/// <summary>
/// A clean start is Started and listening; <c>StopAsync</c> closes the port (connection refused, not
/// a stale-but-listening endpoint) and marks Stopped; <c>DisposeAsync</c> twice afterwards is a no-op.
/// </summary>
[Fact]
public async Task StopAsync_closes_the_port_and_DisposeAsync_is_idempotent()
{
using var deadline = new CancellationTokenSource(Deadline);
var port = FreePort();
var pki = NewPki();
var state = new OpcUaEndpointState();
var service = NewService(port, pki, state, EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null));
try
{
await service.StartAsync(deadline.Token);
state.Status.ShouldBe(OpcUaEndpointStatus.Started);
using (var client = new TcpClient())
{
await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token);
client.Connected.ShouldBeTrue("positive control: the endpoint is listening after StartAsync");
}
await service.StopAsync(deadline.Token);
state.Status.ShouldBe(OpcUaEndpointStatus.Stopped);
using (var client = new TcpClient())
{
await Should.ThrowAsync<SocketException>(async () => await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token));
}
await service.DisposeAsync();
await service.DisposeAsync(); // idempotent
}
finally
{
await service.DisposeAsync();
Cleanup(pki);
}
}
private static OtOpcUaServerHostedService NewService(int port, string pki, OpcUaEndpointState state, EffectiveRoles roles) =>
new(
Options.Create(new OpcUaApplicationHostOptions
{
ApplicationName = "OtOpcUa.StartFailure",
ApplicationUri = $"urn:OtOpcUa.StartFailure:{Guid.NewGuid():N}",
OpcUaPort = port, PublicHostname = "localhost", PkiStoreRoot = pki,
}),
new DeferredAddressSpaceSink(), new DeferredServiceLevelPublisher(),
NullOpcUaUserAuthenticator.Instance, NullHistorianDataSource.Instance,
EmptyDataPlaneAuthoritySource.Instance, new ConfigurationBuilder().Build(),
() => throw new NotSupportedException("no actor system required"), new ActorRegistry(),
null, Options.Create(new AclOptions()), TimeProvider.System, NullLoggerFactory.Instance,
endpointState: state, roles: roles);
/// <summary>The production health registration (driver role) over the supplied state, on a loopback Kestrel.</summary>
private static WebApplication BuildHealthHost(OpcUaEndpointState state)
{
var builder = WebApplication.CreateBuilder();
builder.WebHost.UseUrls("http://127.0.0.1:0");
builder.Logging.ClearProviders();
builder.Services.AddSingleton<IOpcUaEndpointState>(state);
builder.Services.AddOtOpcUaHealth(hasAdmin: false, hasDriver: true);
// The akka + cluster-primary probes need a running ActorSystem; keep only the probe under test so
// the status measured is the endpoint's. (The registration shape itself is pinned separately.)
builder.Services.Configure<HealthCheckServiceOptions>(o =>
{
foreach (var r in o.Registrations.Where(r => r.Name != OpcUaEndpointHealthCheck.Name).ToList())
o.Registrations.Remove(r);
});
var app = builder.Build();
app.MapOtOpcUaHealth();
return app;
}
/// <summary>
/// Holds a port on the IPv4 AND IPv6 wildcard addresses. The SDK binds its listener on the
/// wildcards, and on macOS a loopback-only squatter does not conflict with a wildcard bind (both
/// sockets carry SO_REUSEADDR), so a loopback squatter would let the SDK start — the first run of
/// this test proved exactly that. The SDK also only fails when NEITHER family binds.
/// </summary>
private static TcpListener SquatAllInterfaces(out int port)
{
var squatter = new TcpListener(IPAddress.IPv6Any, 0);
squatter.Server.DualMode = true;
squatter.ExclusiveAddressUse = true;
squatter.Start();
port = ((IPEndPoint)squatter.LocalEndpoint).Port;
return squatter;
}
private static int FreePort()
{
using var probe = new TcpListener(IPAddress.Loopback, 0);
probe.Start();
var port = ((IPEndPoint)probe.LocalEndpoint).Port;
probe.Stop();
return port;
}
private static string NewPki() => Path.Combine(Path.GetTempPath(), $"otopcua-startfail-{Guid.NewGuid():N}");
private static void Cleanup(string pki)
{
if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true);
}
}
@@ -15,6 +15,28 @@ public sealed class DependencyMuxTagUpstreamSourceTests
private static DataValueSnapshot Good(object? value, DateTime ts)
=> new(value, 0u, ts, ts);
/// <summary>R02:L3 — the host prunes the retained snapshots to the loaded dependency-ref union after each
/// (re)load, so a renamed/removed raw path stops being retained; observers are untouched.</summary>
[Fact]
public void RetainOnly_drops_snapshots_outside_the_set_and_keeps_observers()
{
var src = new DependencyMuxTagUpstreamSource();
var ts = DateTime.UtcNow;
src.Push("keep", Good(1, ts));
src.Push("old/renamed", Good(2, ts));
var seen = new List<string>();
using var sub = src.SubscribeTag("old/renamed", (p, _) => seen.Add(p));
src.RetainOnly(new HashSet<string>(StringComparer.Ordinal) { "keep" }).ShouldBe(1);
src.RetainedPathCount.ShouldBe(1);
src.ReadTag("keep").Value.ShouldBe(1);
src.ReadTag("old/renamed").StatusCode.ShouldBe(OpcUaStatus.BadWaitingForInitialData);
src.Push("old/renamed", Good(3, ts));
// Pruning retained values does not unsubscribe anyone.
seen.ShouldBe(new[] { "old/renamed" });
}
[Fact]
public void Push_then_ReadTag_returns_the_pushed_snapshot()
{
@@ -0,0 +1,145 @@
using Akka.Actor;
using Akka.TestKit;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.ScriptedAlarms;
/// <summary>
/// R02:H1 — a supervision restart of <see cref="ScriptedAlarmHostActor"/> used to dispose the
/// Props-captured engine and re-create the actor around the disposed instance, so every later
/// <c>ApplyScriptedAlarms</c> failed and the node's scripted alarms silently died. With the factory
/// Props each incarnation builds its own engine; with the legacy Props a restart only detaches. Both
/// shapes must load cleanly after a throw inside a <see cref="VirtualTagActor.DependencyValueBatch"/>
/// handler. Also R02:H3 — an inbound alarm command's engine op is bounded.
/// </summary>
public sealed class ScriptedAlarmHostActorRestartTests : RuntimeActorTestBase
{
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30);
private static EquipmentScriptedAlarmPlan Plan(string id = "alm-1", string depRef = "M.T") =>
new(ScriptedAlarmId: id, EquipmentId: "Plant/Line1/M", Name: "HighTemp", AlarmType: "AlarmCondition",
Severity: 800, MessageTemplate: "condition", PredicateScriptId: $"{id}-script",
PredicateSource: $"return (int)ctx.GetTag(\"{depRef}\").Value > 90;",
DependencyRefs: new[] { depRef }, HistorizeToAveva: true, Retain: true, Enabled: true);
private static ScriptedAlarmEngine BuildEngine(DependencyMuxTagUpstreamSource upstream, IAlarmStateStore? store = null)
{
var logger = new LoggerConfiguration().CreateLogger();
return new ScriptedAlarmEngine(upstream, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
}
private static (DependencyMuxTagUpstreamSource, ScriptedAlarmEngine) Pair()
{
var up = new DependencyMuxTagUpstreamSource();
return (up, BuildEngine(up));
}
/// <summary>Apply, crash the handler (a null TagId makes <c>Push</c> throw → restart), apply again
/// and prove the second load registers interest and the alarm still activates.</summary>
private void RestartThenReload(IActorRef host, TestProbe publish, TestProbe mux)
{
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
// The crash: a batch whose item has a null TagId makes the upstream Push throw out of the handler
// (the documented, deliberately unguarded path) → supervision restarts the actor.
host.Tell(new VirtualTagActor.DependencyValueBatch(new[]
{
new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow),
}));
mux.ExpectMsg<DependencyMuxActor.UnregisterInterest>(Timeout); // PostStop of the crashed incarnation
// A later apply must load cleanly on the restarted incarnation…
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
var reg = mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
reg.TagRefs.ShouldContain("M.T");
// …and the alarm must still work end to end.
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
var state = publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout);
state.AlarmNodeId.ShouldBe("alm-1");
state.State.Active.ShouldBeTrue();
}
[Fact]
public void Factory_Props_survives_a_handler_throw_and_loads_cleanly_afterwards()
{
var publish = CreateTestProbe();
var mux = CreateTestProbe();
var built = 0;
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, () => { built++; return Pair(); }));
RestartThenReload(host, publish, mux);
built.ShouldBe(2, "the restarted incarnation built its own engine");
}
[Fact]
public void Legacy_instance_Props_survives_a_handler_throw_because_a_restart_only_detaches()
{
var publish = CreateTestProbe();
var mux = CreateTestProbe();
var (upstream, engine) = Pair();
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, upstream, engine));
RestartThenReload(host, publish, mux);
}
[Fact]
public void An_alarm_command_whose_store_hangs_is_bounded_and_the_mailbox_moves_on()
{
var publish = CreateTestProbe();
var mux = CreateTestProbe();
var store = new HangingSaveStore();
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref,
() => { var up = new DependencyMuxTagUpstreamSource(); return (up, BuildEngine(up, store)); },
commandTimeout: TimeSpan.FromMilliseconds(300)));
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout).State.Active.ShouldBeTrue();
store.HangNext = true;
host.Tell(new AlarmCommand("alm-1", "Acknowledge", "alice", "ack", null));
// The hung save would have suspended the mailbox indefinitely; the bound releases it, so the
// next dependency value is processed and the alarm clears.
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 10, DateTime.UtcNow));
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout).State.Active.ShouldBeFalse();
store.ObservedCancellation.ShouldBeTrue();
}
/// <summary>Store whose next SaveAsync blocks until its token is cancelled.</summary>
private sealed class HangingSaveStore : IAlarmStateStore
{
private readonly InMemoryAlarmStateStore _inner = new();
public volatile bool HangNext;
public volatile bool ObservedCancellation;
public Task<AlarmConditionState?> LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct);
public Task<IReadOnlyList<AlarmConditionState>> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct);
public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct);
public async Task SaveAsync(AlarmConditionState state, CancellationToken ct)
{
if (HangNext)
{
HangNext = false;
try { await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, ct); }
catch (OperationCanceledException) { ObservedCancellation = true; throw; }
}
await _inner.SaveAsync(state, ct);
}
}
}
@@ -0,0 +1,184 @@
using System.Collections.Concurrent;
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Calculation;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags;
/// <summary>
/// R07:H6 — the mux retains the last value of every ref a REPLAY-OPTING subscriber
/// (<see cref="DependencyMuxActor.RegisterInterest.ReplayRetained"/>) is registered for, keeps it for a
/// short grace after the last such subscriber leaves, and replays it to a registration that asks. A
/// Calculation driver Restart (new driver + new adapter) therefore republishes from current inputs instead
/// of sitting <c>BadWaitingForInitialData</c> until every input happens to change. Fix round 1 (I2): a
/// non-opting subscriber (the historization recorder) retains nothing, retention is released after the
/// grace, and a removed source's values are evicted so its last Good is never replayed.
/// </summary>
public sealed class DependencyMuxReplayTests : RuntimeActorTestBase
{
private static DriverInstanceActor.AttributeValueBatchPublished Burst(params (string Ref, object? Value)[] items) =>
new("driver-1", items.Select(i => new DriverInstanceActor.AttributeValueItem(i.Ref, i.Value, OpcUaStatus.Good, DateTime.UtcNow)).ToArray());
[Fact]
public void RegisterInterest_with_replay_receives_the_retained_values_for_its_refs_only()
{
var mux = Sys.ActorOf(DependencyMuxActor.Props());
var first = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref, ReplayRetained: true));
mux.Tell(Burst(("a", 1), ("b", 2), ("c", 3)));
first.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget).Items.Count.ShouldBe(2);
// The first subscriber goes away; retention must survive the unregister (that is the Restart shape).
mux.Tell(new DependencyMuxActor.UnregisterInterest(first.Ref));
var second = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "c" }, second.Ref, ReplayRetained: true));
var replay = second.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by a replay-opting subscriber, so it is not retained; 'b' is not in this subscriber's refs");
replay.Items[0].Value.ShouldBe(1);
}
[Fact]
public void RegisterInterest_without_replay_receives_nothing_and_a_fresh_ref_replays_nothing()
{
var mux = Sys.ActorOf(DependencyMuxActor.Props());
var seed = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, seed.Ref));
mux.Tell(Burst(("a", 1)));
seed.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
var opted = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, opted.Ref)); // default: no replay
opted.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
var fresh = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "never-published" }, fresh.Ref, ReplayRetained: true));
fresh.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
}
/// <summary>I2 — a subscriber that did NOT opt into replay (the continuous-historization recorder registers
/// every historized ref) retains nothing; retention is bounded to the refs a replay-opting subscriber named.</summary>
[Fact]
public void Only_refs_named_by_a_replay_opting_subscriber_are_retained()
{
var mux = ActorOfAsTestActorRef<DependencyMuxActor>(DependencyMuxActor.Props());
var recorder = CreateTestProbe();
var calc = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "h1", "h2", "h3", "shared" }, recorder.Ref));
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "r", "shared" }, calc.Ref, ReplayRetained: true));
mux.Tell(Burst(("h1", 1), ("h2", 2), ("h3", 3), ("r", 4), ("shared", 5)));
recorder.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget).Items.Count.ShouldBe(4, "the recorder is still DELIVERED to");
calc.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
mux.UnderlyingActor.RetainedRefCount.ShouldBe(2, "only 'r' and 'shared' — the recorder-only refs retain nothing");
var late = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "h1", "r" }, late.Ref, ReplayRetained: true));
late.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget)
.Items.Select(i => i.TagId).ShouldBe(new[] { "r" }, "positive control: the opted-in ref replays; 'h1' was never retained");
// Unregistering everyone ends up with no entries at all once the grace is over — tested below; here
// the recorder leaving changes nothing about retention.
mux.Tell(new DependencyMuxActor.UnregisterInterest(recorder.Ref));
mux.UnderlyingActor.RetainedRefCount.ShouldBe(2);
}
/// <summary>I2 — when the last replay-opting subscriber of a ref leaves, its retained value survives a short
/// grace (the restart window: old adapter gone, new one not yet registered) and is then dropped.</summary>
[Fact]
public void Retention_is_released_after_the_grace_once_no_replay_subscriber_remains()
{
var mux = ActorOfAsTestActorRef<DependencyMuxActor>(DependencyMuxActor.Props(TimeSpan.FromMilliseconds(200)));
var gen1 = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen1.Ref, ReplayRetained: true));
mux.Tell(Burst(("a", 1)));
gen1.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
mux.Tell(new DependencyMuxActor.UnregisterInterest(gen1.Ref));
// Positive control: inside the grace a returning subscriber still gets the value.
var gen2 = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen2.Ref, ReplayRetained: true));
gen2.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget).Items.Single().Value.ShouldBe(1);
mux.Tell(new DependencyMuxActor.UnregisterInterest(gen2.Ref));
// Past the grace: nothing retained, no entry left behind, and a returning subscriber replays nothing.
AwaitAssert(() =>
{
mux.UnderlyingActor.RetainedRefCount.ShouldBe(0);
mux.UnderlyingActor.TrackedRefCount.ShouldBe(0);
}, PresenceBudget);
var gen3 = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen3.Ref, ReplayRetained: true));
gen3.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
}
/// <summary>I2 — a removed source's retained values are evicted, so its last Good is never replayed to a
/// subscriber that (re)registers after the removal. Values from other sources are untouched.</summary>
[Fact]
public void SourceRemoved_evicts_that_sources_retained_values_only()
{
var mux = ActorOfAsTestActorRef<DependencyMuxActor>(DependencyMuxActor.Props());
var sub = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, sub.Ref, ReplayRetained: true));
mux.Tell(new DriverInstanceActor.AttributeValueBatchPublished("driver-1",
[new DriverInstanceActor.AttributeValueItem("a", 1, OpcUaStatus.Good, DateTime.UtcNow)]));
mux.Tell(new DriverInstanceActor.AttributeValueBatchPublished("driver-2",
[new DriverInstanceActor.AttributeValueItem("b", 2, OpcUaStatus.Good, DateTime.UtcNow)]));
sub.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
sub.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
mux.Tell(new DependencyMuxActor.SourceRemoved("driver-1"));
mux.UnderlyingActor.RetainedRefCount.ShouldBe(1);
var restarted = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, restarted.Ref, ReplayRetained: true));
restarted.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget)
.Items.Select(i => i.TagId).ShouldBe(new[] { "b" }, "driver-1 was removed: its last Good must not be replayed");
}
/// <summary>The user-facing scenario: a calc tag computes, the driver is restarted (new instance + new
/// adapter) with NO input changing afterwards, and the new instance republishes the current value.</summary>
[Fact]
public async Task A_Calculation_driver_restart_republishes_from_the_current_inputs()
{
var mux = Sys.ActorOf(DependencyMuxActor.Props());
var tag = new RawTagEntry("calc/e/sum",
"{\"scriptId\":\"s\",\"changeTriggered\":true,\"scriptSource\":\"return (double)ctx.GetTag(\\\"src/a\\\").Value + (double)ctx.GetTag(\\\"src/b\\\").Value;\"}",
WriteIdempotent: false);
// Generation 1: wire the driver through the real adapter and feed a burst.
var gen1 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv");
var published1 = new ConcurrentQueue<DataChangeEventArgs>();
gen1.OnDataChange += (_, e) => published1.Enqueue(e);
await gen1.InitializeAsync("{}", CancellationToken.None);
var adapter1 = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen1, mux));
AwaitAssert(() =>
{
mux.Tell(Burst(("src/a", 10.0), ("src/b", 20.0)));
published1.Count.ShouldBeGreaterThan(0);
}, PresenceBudget);
published1.Last().Snapshot.Value.ShouldBe(30.0);
// Restart: the old instance goes away; a NEW driver + adapter come up. Nothing changes upstream.
Sys.Stop(adapter1);
await gen1.ShutdownAsync(CancellationToken.None);
var gen2 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv");
var published2 = new ConcurrentQueue<DataChangeEventArgs>();
gen2.OnDataChange += (_, e) => published2.Enqueue(e);
Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen2, mux)); // registers with ReplayRetained
await gen2.InitializeAsync("{}", CancellationToken.None);
// The replayed inputs satisfied the change-gate; the first trigger after init is the replay itself
// (if it lands post-init) or the next change (here: none) — so publish happens from the replay.
AwaitAssert(() => published2.Count.ShouldBeGreaterThan(0), PresenceBudget);
published2.Last().Snapshot.Value.ShouldBe(30.0);
published2.Last().Snapshot.StatusCode.ShouldBe(OpcUaStatus.Good);
}
}
@@ -0,0 +1,56 @@
using System.Collections.Concurrent;
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags;
/// <summary>R02:P6 — the per-failure cluster-wide <see cref="ScriptLogEntry"/> is rate-limited per tag
/// (first N per window, then one summary); the Bad publish and the metric are unaffected.</summary>
public sealed class VirtualTagActorFailureLogRateLimitTests : RuntimeActorTestBase
{
[Fact]
public void Repeated_failures_publish_at_most_the_limit_then_one_summary_when_the_window_rolls()
{
var now = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => now);
var capture = new ConcurrentQueue<ScriptLogEntry>();
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(VirtualTagActor.Props(
"vt-chatty", "broken",
evaluator: new FailingEvaluator("boom"),
publisherFactory: () => new DPSPublisher((_, p) => capture.Enqueue((ScriptLogEntry)p)),
failureLogLimiter: limiter));
for (var i = 0; i < 10; i++)
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", i, DateTime.UtcNow));
// Positive control: the FIRST failure still degrades the node (once per transition, as before).
parent.ExpectMsg<VirtualTagActor.EvaluationResult>(PresenceBudget).StatusCode.ShouldBe(Commons.OpcUa.OpcUaStatus.BadInternalError);
AwaitAssert(() => capture.Count.ShouldBe(3), PresenceBudget);
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
capture.Count.ShouldBe(3, "7 of 10 failures were suppressed from the cluster-wide log");
now = now.AddMinutes(1);
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 99, DateTime.UtcNow));
AwaitAssert(() => capture.Count.ShouldBe(5), PresenceBudget);
var entries = capture.ToArray();
entries[3].Message.ShouldContain("7");
entries[3].Message.ShouldContain("suppressed");
entries[3].Message.ShouldContain("limit 3/window", Case.Sensitive, "the summary names the CONFIGURED limit, not the default");
entries[3].VirtualTagId.ShouldBe("vt-chatty");
entries[4].Message.ShouldBe("boom");
}
private sealed class FailingEvaluator(string reason) : IVirtualTagEvaluator
{
public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary<string, object?> deps)
=> VirtualTagEvalResult.Failure(reason);
}
}
@@ -0,0 +1,116 @@
using System.Security.Claims;
using Microsoft.AspNetCore.Components.Authorization;
using Microsoft.AspNetCore.Http;
using Shouldly;
using Xunit;
using ZB.MOM.WW.Auth.AspNetCore;
using ZB.MOM.WW.OtOpcUa.Security.Audit;
namespace ZB.MOM.WW.OtOpcUa.Security.Tests.Audit;
/// <summary>
/// R03:2.5 — inside a Blazor circuit there is no <c>HttpContext</c> after the first render, so an
/// HTTP-only accessor stamps every audit row from an interactive page <c>"system"</c>. The circuit's
/// <see cref="AuthenticationStateProvider"/> is consulted first; the HTTP context is the fallback.
/// </summary>
public sealed class AuthenticationStateAuditActorAccessorTests
{
[Fact]
public void Circuit_principal_wins_when_there_is_no_http_context()
{
var accessor = new AuthenticationStateAuditActorAccessor(
new HttpContextAccessorStub(null),
new FixedStateProvider(Authenticated(new Claim(ZbClaimTypes.Username, "alice"))));
accessor.CurrentActor.ShouldBe("alice");
}
[Fact]
public void Http_principal_is_the_fallback_outside_a_circuit()
{
var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "bob")) };
var accessor = new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), authenticationState: null);
accessor.CurrentActor.ShouldBe("bob");
}
[Fact]
public void Unauthenticated_circuit_falls_through_to_http_then_null()
{
var anonymousCircuit = new FixedStateProvider(new ClaimsPrincipal(new ClaimsIdentity()));
new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(null), anonymousCircuit)
.CurrentActor.ShouldBeNull();
var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Name, "carol")) };
new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), anonymousCircuit)
.CurrentActor.ShouldBe("carol");
}
/// <summary>An incomplete state task is "no circuit identity", never a block.</summary>
[Fact]
public void Pending_authentication_state_is_not_awaited()
{
var accessor = new AuthenticationStateAuditActorAccessor(
new HttpContextAccessorStub(null),
new PendingStateProvider());
accessor.CurrentActor.ShouldBeNull();
}
/// <summary>
/// Fix round 1 / I5 — the REAL server-side provider, resolved in a non-circuit scope (a minimal-API
/// request, a background service), throws <see cref="InvalidOperationException"/> from
/// <c>GetAuthenticationStateAsync</c> because nothing called <c>SetAuthenticationState</c>. That used
/// to escape <c>CurrentActor</c>; it must fall back to the HTTP principal instead.
/// </summary>
[Fact]
public void Real_server_provider_outside_a_circuit_falls_back_to_the_http_principal()
{
var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "dave")) };
var accessor = new AuthenticationStateAuditActorAccessor(
new HttpContextAccessorStub(context),
new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider());
accessor.CurrentActor.ShouldBe("dave");
// No HTTP principal either ⇒ null, still no throw.
new AuthenticationStateAuditActorAccessor(
new HttpContextAccessorStub(null),
new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider())
.CurrentActor.ShouldBeNull();
}
/// <summary>Positive control on the real provider: once a circuit has set its state, that principal wins.</summary>
[Fact]
public void Real_server_provider_inside_a_circuit_serves_the_circuit_principal()
{
var provider = new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider();
provider.SetAuthenticationState(Task.FromResult(
new AuthenticationState(Authenticated(new Claim(ZbClaimTypes.Username, "erin")))));
var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "http-user")) };
new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), provider)
.CurrentActor.ShouldBe("erin");
}
private static ClaimsPrincipal Authenticated(params Claim[] claims) =>
new(new ClaimsIdentity(claims, "TestScheme", ZbClaimTypes.Name, ZbClaimTypes.Role));
private sealed class HttpContextAccessorStub(HttpContext? context) : IHttpContextAccessor
{
public HttpContext? HttpContext { get; set; } = context;
}
private sealed class FixedStateProvider(ClaimsPrincipal user) : AuthenticationStateProvider
{
public override Task<AuthenticationState> GetAuthenticationStateAsync() =>
Task.FromResult(new AuthenticationState(user));
}
private sealed class PendingStateProvider : AuthenticationStateProvider
{
public override Task<AuthenticationState> GetAuthenticationStateAsync() =>
new TaskCompletionSource<AuthenticationState>().Task;
}
}
@@ -11,6 +11,8 @@ using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.IdentityModel.Tokens;
using Shouldly;
using Xunit;
using ZB.MOM.WW.Auth.AspNetCore;
@@ -37,6 +39,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime
private IHost _host = null!;
private TestServer _server = null!;
private readonly StubLdapGroupRoleMappingService _roleMappings = new();
private readonly CapturingLoggerProvider _logs = new();
private static CancellationToken Ct => TestContext.Current.CancellationToken;
@@ -70,6 +73,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime
["Security:Ldap:GroupToRole:ReadOnly"] = "Viewer",
}).Build();
services.AddOtOpcUaAuth(configuration);
services.AddLogging(l => l.AddProvider(_logs));
services.AddSingleton<ILdapAuthService, StubLdapAuthService>();
// The login handler now resolves the DB role-map service via DI to merge
// DB-backed grants on top of the appsettings baseline. Register the stub so
@@ -96,6 +100,10 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime
.ToArray();
return Results.Ok(claims);
}).RequireAuthorization();
// A role-gated machine endpoint: proves a bearer principal's roles resolve through
// IsInRole (the validation parameters pair RoleClaimType with the minted key).
e.MapGet("/api/viewer-only", () => Results.Ok("viewer"))
.RequireAuthorization(p => p.RequireRole("Viewer"));
});
});
})
@@ -153,6 +161,158 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime
new AuthEndpoints.LoginRequest("ldap-down", "anything"), Ct);
response.StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable);
// R03:2.7 — the outage is named in the server log, not inferred from the directory side.
_logs.Entries.Any(e =>
e.Level == LogLevel.Warning && e.Message.Contains("LDAP authentication failed") && e.Exception != null)
.ShouldBeTrue("a Warning naming the LDAP failure must be logged");
}
/// <summary>
/// R03:2.4 — a token from <c>/auth/token</c> authenticates a request carrying it as a bearer:
/// the fallback-protected root AND a role-gated endpoint (the bearer principal's "Role" claims
/// resolve through <c>IsInRole</c>). No cookie is sent.
/// </summary>
[Fact]
public async Task Bearer_token_authenticates_api_requests_without_a_cookie()
{
var token = await LoginAndIssueTokenAsync();
var client = NewClient();
var root = new HttpRequestMessage(HttpMethod.Get, "/");
root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
(await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK);
var gated = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only");
gated.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
var gatedResponse = await client.SendAsync(gated, Ct);
gatedResponse.StatusCode.ShouldBe(HttpStatusCode.OK);
(await gatedResponse.Content.ReadAsStringAsync(Ct)).ShouldContain("viewer");
}
/// <summary>A tampered or garbage bearer fails closed (401), it never falls through to anonymous.</summary>
[Theory]
[InlineData("not-a-jwt")]
[InlineData("tampered")]
public async Task Bad_bearer_token_is_rejected(string kind)
{
var token = kind == "tampered"
? (await LoginAndIssueTokenAsync()) + "x"
: kind;
var client = NewClient();
var req = new HttpRequestMessage(HttpMethod.Get, "/");
req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
req.Headers.Add("X-Requested-With", "XMLHttpRequest");
var response = await client.SendAsync(req, Ct);
response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized);
response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer");
}
/// <summary>
/// Fix round 1 / I1 — the first-party JwtBearer handler pins HS256: an unsigned (<c>alg: none</c>)
/// token carrying the right issuer/audience/roles, and a token signed with a different HMAC
/// variant, both fail CLOSED with a 401 — never a 500, never a pass.
/// </summary>
[Theory]
[InlineData("none")]
[InlineData("HS384")]
public async Task Bearer_with_a_non_HS256_algorithm_is_rejected_with_401(string alg)
{
var token = alg == "none" ? UnsignedToken() : SignedToken(SecurityAlgorithms.HmacSha384);
var client = NewClient();
var req = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only");
req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
var response = await client.SendAsync(req, Ct);
response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized);
response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer");
// Positive control: the same claims signed HS256 with the configured key ARE accepted, so the
// 401 above measured the algorithm and not a broken token shape.
var good = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only");
good.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue(
"Bearer", SignedToken(SecurityAlgorithms.HmacSha256));
(await client.SendAsync(good, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK);
}
/// <summary>
/// Fix round 1 / I2 — <c>/auth/token</c> mints only for a COOKIE session. A bearer token used to
/// be able to mint a fresh token for itself, so a stolen token could renew itself forever and
/// outlive both its own expiry and the user's LDAP membership. Positive control: the cookie
/// session that minted the bearer still gets a token.
/// </summary>
[Fact]
public async Task Auth_token_refuses_a_bearer_only_caller_and_serves_a_cookie_session()
{
var client = NewClient();
var loginResponse = await client.PostAsJsonAsync("/auth/login",
new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct);
loginResponse.EnsureSuccessStatusCode();
var cookieReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token");
AttachCookies(cookieReq, loginResponse);
var cookieResp = await client.SendAsync(cookieReq, Ct);
cookieResp.StatusCode.ShouldBe(HttpStatusCode.OK);
var bearer = (await cookieResp.Content.ReadFromJsonAsync<JsonElement>(Ct)).GetProperty("token").GetString()!;
// The bearer authenticates elsewhere (positive control on the token itself) …
var root = new HttpRequestMessage(HttpMethod.Get, "/");
root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer);
(await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK);
// … but cannot renew itself.
var renew = new HttpRequestMessage(HttpMethod.Post, "/auth/token");
renew.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer);
renew.Headers.Add("X-Requested-With", "XMLHttpRequest");
var renewResp = await client.SendAsync(renew, Ct);
renewResp.StatusCode.ShouldBeOneOf(HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden);
(await renewResp.Content.ReadAsStringAsync(Ct)).ShouldNotContain("token");
}
private const string TestSigningKey = "test-signing-key-with-at-least-32-bytes-of-utf8-content";
private static IEnumerable<System.Security.Claims.Claim> ViewerClaims() =>
[
new(JwtTokenService.UsernameClaimType, "mallory"),
new(JwtTokenService.RoleClaimType, "Viewer"),
];
/// <summary>A token signed with <paramref name="algorithm"/> using the configured key bytes.</summary>
private static string SignedToken(string algorithm)
{
var keyBytes = System.Text.Encoding.UTF8.GetBytes(TestSigningKey);
// The SAME key bytes for every algorithm, so a rejection measures the algorithm pin alone.
var token = new System.IdentityModel.Tokens.Jwt.JwtSecurityToken(
issuer: "otopcua-test", audience: "otopcua-test", claims: ViewerClaims(),
expires: DateTime.UtcNow.AddMinutes(5),
signingCredentials: new SigningCredentials(new SymmetricSecurityKey(keyBytes), algorithm));
return new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler().WriteToken(token);
}
/// <summary>An <c>alg: none</c> token with otherwise valid issuer/audience/lifetime/roles.</summary>
private static string UnsignedToken()
{
static string B64(string json) => Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(json);
var exp = DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeSeconds();
return B64("{\"alg\":\"none\",\"typ\":\"JWT\"}") + "." +
B64($"{{\"zb:username\":\"mallory\",\"Role\":\"Viewer\",\"iss\":\"otopcua-test\",\"aud\":\"otopcua-test\",\"exp\":{exp}}}") +
".";
}
private async Task<string> LoginAndIssueTokenAsync()
{
var client = NewClient();
var loginResponse = await client.PostAsJsonAsync("/auth/login",
new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct);
loginResponse.EnsureSuccessStatusCode();
var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token");
AttachCookies(tokenReq, loginResponse);
var response = await client.SendAsync(tokenReq, Ct);
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadFromJsonAsync<JsonElement>(Ct);
return payload.GetProperty("token").GetString()!;
}
/// <summary>Tests that ping anonymous returns 401.</summary>
@@ -0,0 +1,25 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
namespace ZB.MOM.WW.OtOpcUa.Security.Tests;
/// <summary>Collects every log entry written through the host's logger factory, for assertions on what was logged.</summary>
internal sealed class CapturingLoggerProvider : ILoggerProvider
{
public ConcurrentQueue<Entry> Entries { get; } = new();
public ILogger CreateLogger(string categoryName) => new Logger(categoryName, Entries);
public void Dispose() { }
internal sealed record Entry(string Category, LogLevel Level, string Message, Exception? Exception);
private sealed class Logger(string category, ConcurrentQueue<Entry> entries) : ILogger
{
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) =>
entries.Enqueue(new Entry(category, logLevel, formatter(state, exception), exception));
}
}
@@ -0,0 +1,127 @@
using System.Net;
using System.Net.Http.Json;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.TestHost;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Services;
using ZB.MOM.WW.OtOpcUa.Security.Auth;
using ZB.MOM.WW.OtOpcUa.Security.Endpoints;
using ZB.MOM.WW.OtOpcUa.Security.Jwt;
using ZB.MOM.WW.OtOpcUa.Security.Ldap;
namespace ZB.MOM.WW.OtOpcUa.Security.Tests;
/// <summary>
/// R03:2.4 — JWT is opt-in by signing key. Unconfigured: <c>/auth/token</c> is not mapped (it used to
/// answer HTTP 500 from the service constructor) and no bearer scheme exists. Configured badly: the
/// host refuses to START rather than failing on the first token request.
/// </summary>
public sealed class JwtConfigurationGateTests
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
[Fact]
public async Task Without_a_signing_key_auth_token_is_not_mapped_and_bearer_is_ignored()
{
using var host = await StartHostAsync(signingKey: null);
var server = host.GetTestServer();
var client = server.CreateClient();
// The login still works — the cookie path is untouched.
var login = await client.PostAsJsonAsync("/auth/login", new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct);
login.StatusCode.ShouldBe(HttpStatusCode.NoContent);
// /auth/token is simply absent (404), not a 500.
var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token");
foreach (var c in login.Headers.GetValues("Set-Cookie")) tokenReq.Headers.Add("Cookie", c.Split(';')[0]);
(await client.SendAsync(tokenReq, Ct)).StatusCode.ShouldBe(HttpStatusCode.NotFound);
// No bearer scheme is registered, so a bearer header is ignored and the cookie session still wins.
var schemes = host.Services.GetRequiredService<IAuthenticationSchemeProvider>();
(await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer)).ShouldBeNull();
var withBogusBearer = new HttpRequestMessage(HttpMethod.Get, "/");
foreach (var c in login.Headers.GetValues("Set-Cookie")) withBogusBearer.Headers.Add("Cookie", c.Split(';')[0]);
withBogusBearer.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "garbage");
(await client.SendAsync(withBogusBearer, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK);
await host.StopAsync(Ct);
}
/// <summary>Positive control for the gate: with a key, the bearer scheme IS registered.</summary>
[Fact]
public async Task With_a_signing_key_the_bearer_scheme_is_registered()
{
using var host = await StartHostAsync(signingKey: "test-signing-key-with-at-least-32-bytes-of-utf8-content");
var schemes = host.Services.GetRequiredService<IAuthenticationSchemeProvider>();
var bearer = await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer);
bearer.ShouldNotBeNull();
bearer.HandlerType.ShouldBe(typeof(Microsoft.AspNetCore.Authentication.JwtBearer.JwtBearerHandler));
await host.StopAsync(Ct);
}
/// <summary>A configured-but-short key fails at host start (ValidateOnStart), not on the first /auth/token.</summary>
[Fact]
public async Task A_short_signing_key_fails_host_start()
{
var ex = await Should.ThrowAsync<OptionsValidationException>(async () =>
{
using var host = await StartHostAsync(signingKey: "too-short");
});
ex.Message.ShouldContain("SigningKey");
}
private static async Task<IHost> StartHostAsync(string? signingKey)
{
var dbName = $"jwt-gate-{Guid.NewGuid():N}";
var settings = new Dictionary<string, string?>();
if (signingKey is not null) settings["Security:Jwt:SigningKey"] = signingKey;
var configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build();
var host = new HostBuilder()
.ConfigureWebHost(web =>
{
web.UseTestServer();
web.ConfigureServices(services =>
{
services.AddDbContextFactory<OtOpcUaConfigDbContext>(opt => opt.UseInMemoryDatabase(dbName));
services.AddDbContext<OtOpcUaConfigDbContext>(opt => opt.UseInMemoryDatabase(dbName));
services.AddRouting();
services.AddOtOpcUaAuth(configuration);
services.AddSingleton<ILdapAuthService, AlwaysAliceLdap>();
services.AddSingleton<ILdapGroupRoleMappingService, NullLdapGroupRoleMappingService>();
});
web.Configure(app =>
{
app.UseRouting();
app.UseAuthentication();
app.UseAuthorization();
app.UseEndpoints(e =>
{
e.MapOtOpcUaAuth();
e.MapGet("/", () => Results.Ok("authenticated")).RequireAuthorization();
});
});
})
.Build();
await host.StartAsync(Ct);
return host;
}
private sealed class AlwaysAliceLdap : ILdapAuthService
{
public Task<LdapAuthResult> AuthenticateAsync(string username, string password, CancellationToken ct = default) =>
Task.FromResult(new LdapAuthResult(
Success: password == "valid-password", DisplayName: "Alice", Username: username,
Groups: [], Roles: [], Error: null));
}
}