fix(alarms): #647 re-review — re-statement needs the same message; CNC refusal is not comms loss; worst-of health
R1: NativeAlarmProjector treats Raise-on-Active as a re-statement only when the message matches what
the condition shows; a different alarm appearing during a respawn un-acks and writes its alerts row.
FOCAS condition messages are a deterministic function of the matched set (tested across fresh driver
instances and reordered lists); Runtime E2E now covers both the same alarm and a new one during a
respawn. Residual (source message determinism) documented in AlarmTracking.md.
R2: a non-OK cnc_rdalmmsg2 return is thrown as FocasWireException{Rc} and classified as a CNC
refusal: conditions get a quality-only BadNotSupported once, one Warning, polling continues, and
driver health is untouched (no 90 s watchdog reconnect loop on a production control). Only transport
faults degrade (falsification control in the test).
R3: alarm-path health is tracked apart from the value path and combined in GetHealth: worst state,
later LastSuccessfulRead. A good value read can no longer mask a failing alarm read (no per-poll
/hosts flip) and a good alarm read cannot mask a value Degraded.
Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
@@ -126,6 +126,9 @@ internal class FakeFocasClient : IFocasClient
|
||||
/// <summary>Gets or sets a value indicating whether the alarm read throws (Gitea #647: a failed read).</summary>
|
||||
public bool ThrowOnReadAlarms { get; set; }
|
||||
|
||||
/// <summary>Gets or sets an exception the alarm read throws (e.g. a CNC refusal carrying an rc).</summary>
|
||||
public Exception? AlarmReadException { get; set; }
|
||||
|
||||
/// <summary>Gets the number of alarm reads served (successful or not).</summary>
|
||||
public int AlarmReadCount => Volatile.Read(ref _alarmReadCount);
|
||||
|
||||
@@ -143,6 +146,7 @@ internal class FakeFocasClient : IFocasClient
|
||||
public virtual Task<IReadOnlyList<FocasActiveAlarm>> ReadAlarmsAsync(CancellationToken ct)
|
||||
{
|
||||
Interlocked.Increment(ref _alarmReadCount);
|
||||
if (AlarmReadException is { } refusal) throw refusal;
|
||||
if (ThrowOnReadAlarms) throw new InvalidOperationException("cnc_rdalmmsg2 failed (fake)");
|
||||
lock (Alarms) return Task.FromResult<IReadOnlyList<FocasActiveAlarm>>([.. Alarms]);
|
||||
}
|
||||
|
||||
@@ -545,6 +545,111 @@ public sealed class FocasAlarmProjectionTests
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Gitea #647 re-review R1–R3 -------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// R1 — the host treats a respawned driver's Raise as a re-statement only when its message matches what the
|
||||
/// condition shows, so the message must be a deterministic function of the matched SET: two fresh driver
|
||||
/// instances reading the same alarms in a different wire (and axis) order say exactly the same thing, and a
|
||||
/// different set says something different.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task The_condition_message_is_deterministic_for_the_same_alarm_set_across_driver_instances()
|
||||
{
|
||||
async Task<string> FirstRaiseMessage(params FocasActiveAlarm[] alarms)
|
||||
{
|
||||
var (drv, factory) = NewDriver(Tag(AnyPath, "ALARM:*"));
|
||||
await InitAndSubscribeForTicksAsync(drv, factory, AnyPath);
|
||||
var events = Capture(drv);
|
||||
Tick(drv, alarms);
|
||||
return events.Single().Message;
|
||||
}
|
||||
|
||||
var a = await FirstRaiseMessage(Servo401(axis: 2), Overheat700, Servo401(axis: 1), Overtravel506);
|
||||
var b = await FirstRaiseMessage(Overtravel506, Servo401(axis: 1), Overheat700, Servo401(axis: 2));
|
||||
b.ShouldBe(a);
|
||||
(await FirstRaiseMessage(Overtravel506, Servo401(axis: 1))).ShouldNotBe(a, "a different set must not read as a re-statement");
|
||||
}
|
||||
|
||||
private static Wire.FocasWireException CncRefusal() => new("cnc_rdalmmsg2 refused EW_FUNC", rc: 1);
|
||||
|
||||
/// <summary>
|
||||
/// R2 — a CNC REFUSAL of cnc_rdalmmsg2 (an rc; the transport is fine) is a capability answer, not comms loss:
|
||||
/// health stays Healthy (so the #527 watchdog can never reconnect-loop a production control), the conditions
|
||||
/// are annotated BadNotSupported once, one Warning is logged, and polling continues at the normal cadence.
|
||||
/// Falsification control: a TRANSPORT failure on the same driver does degrade.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_CNC_refusal_marks_conditions_not_supported_without_degrading_health()
|
||||
{
|
||||
var logger = new ListLogger();
|
||||
var factory = new FakeFocasClientFactory { Customise = () => new FakeFocasClient { AlarmReadException = CncRefusal() } };
|
||||
var drv = new FocasDriver(new FocasDriverOptions
|
||||
{
|
||||
Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")],
|
||||
RawTags = [Tag(ServoPath, "ALARM:Servo")],
|
||||
Probe = new FocasProbeOptions { Enabled = false },
|
||||
AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) },
|
||||
}, "drv-1", factory, logger);
|
||||
await drv.InitializeAsync("{}", CancellationToken.None);
|
||||
var events = Capture(drv);
|
||||
await drv.SubscribeAlarmsAsync([ServoPath], CancellationToken.None);
|
||||
await WaitFor(() => factory.Clients.Count > 0 && factory.Clients[0].AlarmReadCount >= 5);
|
||||
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a refusal is not a comms fault");
|
||||
lock (events)
|
||||
{
|
||||
events.Count.ShouldBe(1);
|
||||
events[0].IsQualityOnly.ShouldBeTrue();
|
||||
events[0].QualityStatusCode.ShouldBe(FocasStatusMapper.BadNotSupported);
|
||||
}
|
||||
logger.Warnings().ShouldBe(1);
|
||||
factory.Clients.Count.ShouldBe(1, "a refusal does not tear the session down");
|
||||
|
||||
// Control: the same driver losing its transport DOES degrade, and re-annotates the condition.
|
||||
factory.Clients[0].AlarmReadException = null;
|
||||
factory.Clients[0].ThrowOnReadAlarms = true;
|
||||
await WaitFor(() => drv.GetHealth().State == DriverState.Degraded);
|
||||
lock (events) events[^1].QualityStatusCode.ShouldBe(BadNoCommunication);
|
||||
await drv.ShutdownAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// R3 — on a mixed driver health is the WORST of the value path and the alarm path, computed together: a
|
||||
/// failing alarm read keeps the driver Degraded however many value reads succeed (no per-poll flip on
|
||||
/// /hosts), and a successful alarm read cannot mask a value-path Degraded.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Health_is_the_worst_of_the_value_and_alarm_paths()
|
||||
{
|
||||
const string valuePath = "Plant/Cnc/m1/r100";
|
||||
var (drv, factory) = NewDriver(
|
||||
new RawTagEntry(valuePath, """{"address":"R100","dataType":"Byte"}""", false, "m1"),
|
||||
Tag(ServoPath, "ALARM:Servo"));
|
||||
await drv.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
// Value reads good, alarm path failing ⇒ Degraded, and it stays Degraded across further good value reads.
|
||||
(await drv.ReadAsync([valuePath], CancellationToken.None))[0].StatusCode.ShouldBe(FocasStatusMapper.Good);
|
||||
drv.NoteAlarmReadFailed(Host, new InvalidOperationException("socket closed"));
|
||||
for (var i = 0; i < 3; i++)
|
||||
{
|
||||
await drv.ReadAsync([valuePath], CancellationToken.None);
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Degraded, "a good value read must not mask the failing alarm path");
|
||||
}
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldNotBeNull("value reads still succeed, so the watchdog is not tripped");
|
||||
|
||||
// Alarm path recovers ⇒ Healthy again.
|
||||
drv.NoteAlarmReadSucceeded(Host);
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
|
||||
// Value path Degraded, alarm path good ⇒ Degraded: an alarm success cannot mask it.
|
||||
factory.Clients.Single().ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange;
|
||||
await drv.ReadAsync([valuePath], CancellationToken.None);
|
||||
drv.NoteAlarmReadSucceeded(Host);
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
|
||||
await drv.ShutdownAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
private static int Count(List<AlarmEventArgs> events)
|
||||
{
|
||||
lock (events) return events.Count;
|
||||
|
||||
Reference in New Issue
Block a user