fix(alarms): #647 re-review — re-statement needs the same message; CNC refusal is not comms loss; worst-of health
R1: NativeAlarmProjector treats Raise-on-Active as a re-statement only when the message matches what
the condition shows; a different alarm appearing during a respawn un-acks and writes its alerts row.
FOCAS condition messages are a deterministic function of the matched set (tested across fresh driver
instances and reordered lists); Runtime E2E now covers both the same alarm and a new one during a
respawn. Residual (source message determinism) documented in AlarmTracking.md.
R2: a non-OK cnc_rdalmmsg2 return is thrown as FocasWireException{Rc} and classified as a CNC
refusal: conditions get a quality-only BadNotSupported once, one Warning, polling continues, and
driver health is untouched (no 90 s watchdog reconnect loop on a production control). Only transport
faults degrade (falsification control in the test).
R3: alarm-path health is tracked apart from the value path and combined in GetHealth: worst state,
later LastSuccessfulRead. A good value read can no longer mask a failing alarm read (no per-poll
/hosts flip) and a good alarm read cannot mask a value Degraded.
Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
@@ -57,22 +57,37 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
|
||||
/// <summary>Devices whose last read came back at the per-read cap. Guarded by <see cref="_alarmLock"/>.</summary>
|
||||
private readonly HashSet<string> _truncationWarned = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Devices whose alarm read is currently failing (one Warning per episode). Guarded by <see cref="_alarmLock"/>.</summary>
|
||||
private readonly HashSet<string> _failingDevices = new(StringComparer.OrdinalIgnoreCase);
|
||||
/// <summary>
|
||||
/// Devices whose alarm read is currently failing → the quality that failure maps to (one Warning per episode
|
||||
/// and per kind: a refusal turning into a transport fault warns again). Guarded by <see cref="_alarmLock"/>.
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, uint> _failingDevices = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// ConditionIds this projection has annotated <c>BadNoCommunication</c> and not yet lifted. Guarded by
|
||||
/// <see cref="_alarmLock"/>. Kept apart from <see cref="_told"/>: a quality annotation never touches a
|
||||
/// condition's Active/Acked state.
|
||||
/// </summary>
|
||||
private readonly HashSet<string> _qualityBadTold = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, uint> _qualityBadTold = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Canonical <c>Opc.Ua.StatusCodes.Good</c>.</summary>
|
||||
private const uint QualityGood = 0x00000000u;
|
||||
|
||||
/// <summary>Canonical <c>Opc.Ua.StatusCodes.BadNoCommunication</c>.</summary>
|
||||
/// <summary>Canonical <c>Opc.Ua.StatusCodes.BadNoCommunication</c> — the CNC could not be reached.</summary>
|
||||
private const uint QualityBadNoCommunication = 0x80310000u;
|
||||
|
||||
/// <summary>
|
||||
/// Whether an alarm-read failure is a CNC REFUSAL (Gitea #647 re-review R2): the CNC answered with a non-OK
|
||||
/// FOCAS return code, so the transport is fine and the control simply will not serve <c>cnc_rdalmmsg2</c>
|
||||
/// (e.g. <c>EW_FUNC</c>, as <c>cnc_rdparam</c> does on the Makino). That is a capability answer, not comms
|
||||
/// loss: it must NOT degrade driver health, or an alarm-only driver on such a control would be reconnected by
|
||||
/// the #527 watchdog every 90 s forever. Everything else (session down, wire fault, malformed reply, backoff)
|
||||
/// is a transport failure.
|
||||
/// </summary>
|
||||
/// <param name="error">The alarm-read failure.</param>
|
||||
/// <returns><see langword="true"/> for a refusal.</returns>
|
||||
internal static bool IsCncRefusal(Exception error) => error is Wire.FocasWireException { Rc: not null };
|
||||
|
||||
/// <summary>Serializes loop start/stop.</summary>
|
||||
private readonly SemaphoreSlim _loopGate = new(1, 1);
|
||||
|
||||
@@ -246,13 +261,15 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Not "no alarms": keep every condition on this device exactly as the server was last told, but say
|
||||
// its source is unreachable — and let the failure reach driver health (an alarm-only driver has no
|
||||
// other read to trip the comms-loss watchdog with).
|
||||
_driver.NoteAlarmReadFailed(host, ex);
|
||||
// why it cannot be read. A transport failure reaches driver health (an alarm-only driver has no other
|
||||
// read to trip the comms-loss watchdog with); a CNC refusal does not (R2). Polling continues at the
|
||||
// normal cadence either way.
|
||||
if (!IsCncRefusal(ex)) _driver.NoteAlarmReadFailed(host, ex);
|
||||
else _driver.NoteAlarmReadRefused(host); // the CNC answered: the transport to it is fine
|
||||
TickFailed(sub, host, tags, ex);
|
||||
continue;
|
||||
}
|
||||
_driver.NoteAlarmReadSucceeded();
|
||||
_driver.NoteAlarmReadSucceeded(host);
|
||||
Tick(sub, host, alarms, tags);
|
||||
}
|
||||
}
|
||||
@@ -290,14 +307,14 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
|
||||
var handlers = _driver.AlarmHandlers;
|
||||
if (handlers is null) return;
|
||||
|
||||
if (_failingDevices.Remove(deviceHostAddress))
|
||||
if (_failingDevices.Remove(deviceHostAddress, out _))
|
||||
_logger.LogInformation("FOCAS alarm read for {Host} recovered", deviceHostAddress);
|
||||
|
||||
foreach (var tag in deviceTags.Where(t => sub.References.Contains(t.RawPath)))
|
||||
{
|
||||
// Lift a Bad this projection put there BEFORE the normal diff, so a recovery that also changed the
|
||||
// alarm state (cleared during the outage) lands as Good then Clear.
|
||||
if (_qualityBadTold.Remove(tag.RawPath))
|
||||
if (_qualityBadTold.Remove(tag.RawPath, out _))
|
||||
Deliver(handlers, QualityOnly(tag, sub.Handle, QualityGood, now), tag.RawPath);
|
||||
|
||||
var view = FocasAlarmMatcher.Evaluate(tag.Selector, alarms, deviceSelectors, truncated);
|
||||
@@ -312,8 +329,9 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
|
||||
|
||||
/// <summary>
|
||||
/// A FAILED read of <paramref name="deviceHostAddress"/> (Gitea #647 review): conditions keep their state, but
|
||||
/// each subscribed one is annotated <c>BadNoCommunication</c> once per outage through the #626 quality-only
|
||||
/// path (no Active/Acked change, no <c>/alerts</c> row), and the first failure of the episode logs one
|
||||
/// each subscribed one is annotated once per outage through the #626 quality-only path (no Active/Acked change,
|
||||
/// no <c>/alerts</c> row) — <c>BadNoCommunication</c> for a transport failure, <c>BadNotSupported</c> for a CNC
|
||||
/// refusal (R2, <see cref="IsCncRefusal"/>) — and the first failure of the episode (per kind) logs one
|
||||
/// Warning. The next successful read lifts the Bad in <see cref="Tick"/>.
|
||||
/// </summary>
|
||||
/// <param name="sub">The subscription the read was taken for; a superseded one emits nothing.</param>
|
||||
@@ -328,10 +346,21 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
|
||||
{
|
||||
if (!ReferenceEquals(Volatile.Read(ref _current), sub)) return;
|
||||
|
||||
if (_failingDevices.Add(deviceHostAddress))
|
||||
_logger.LogWarning(error,
|
||||
"FOCAS alarm read (cnc_rdalmmsg2) failed for {Host}; its alarm conditions keep their state and are " +
|
||||
"marked BadNoCommunication until a read succeeds", deviceHostAddress);
|
||||
var refusal = IsCncRefusal(error);
|
||||
var quality = refusal ? FocasStatusMapper.BadNotSupported : QualityBadNoCommunication;
|
||||
if (!_failingDevices.TryGetValue(deviceHostAddress, out var was) || was != quality)
|
||||
{
|
||||
_failingDevices[deviceHostAddress] = quality;
|
||||
if (refusal)
|
||||
_logger.LogWarning(error,
|
||||
"FOCAS CNC {Host} refuses the alarm read (cnc_rdalmmsg2); its alarm conditions keep their state " +
|
||||
"and are marked BadNotSupported. This is a capability answer, not comms loss — driver health is " +
|
||||
"unaffected and polling continues", deviceHostAddress);
|
||||
else
|
||||
_logger.LogWarning(error,
|
||||
"FOCAS alarm read (cnc_rdalmmsg2) failed for {Host}; its alarm conditions keep their state and " +
|
||||
"are marked BadNoCommunication until a read succeeds", deviceHostAddress);
|
||||
}
|
||||
else
|
||||
_logger.LogDebug(error, "FOCAS alarm read for {Host} still failing", deviceHostAddress);
|
||||
|
||||
@@ -343,8 +372,9 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
|
||||
string.Equals(t.DeviceHostAddress, deviceHostAddress, StringComparison.OrdinalIgnoreCase)
|
||||
&& sub.References.Contains(t.RawPath)))
|
||||
{
|
||||
if (!_qualityBadTold.Add(tag.RawPath)) continue;
|
||||
Deliver(handlers, QualityOnly(tag, sub.Handle, QualityBadNoCommunication, now), tag.RawPath);
|
||||
if (_qualityBadTold.TryGetValue(tag.RawPath, out var told) && told == quality) continue;
|
||||
_qualityBadTold[tag.RawPath] = quality;
|
||||
Deliver(handlers, QualityOnly(tag, sub.Handle, quality, now), tag.RawPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,6 +200,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
// AdoptConfigDuringInit calls this without a teardown first). Not ShutdownAsync: that also drops _poll
|
||||
// and subscriptions, which are owned by the shutdown path (the actor re-subscribes after every successful init).
|
||||
await ReleaseDeviceStatesAsync().ConfigureAwait(false);
|
||||
ResetAlarmHealth();
|
||||
Volatile.Write(ref _health, new DriverHealth(DriverState.Initializing, null, null));
|
||||
try
|
||||
{
|
||||
@@ -475,11 +476,32 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
_resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry
|
||||
_parsedAddressesByTagName.Clear();
|
||||
_fixedTreeActive = false;
|
||||
ResetAlarmHealth();
|
||||
Volatile.Write(ref _health, new DriverHealth(DriverState.Unknown, Volatile.Read(ref _health).LastSuccessfulRead, null));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public DriverHealth GetHealth() => Volatile.Read(ref _health);
|
||||
/// <remarks>
|
||||
/// The worst of the value path (<see cref="_health"/>) and the alarm path (<see cref="_alarmTransportFailures"/>),
|
||||
/// computed together (Gitea #647 re-review R3). <see cref="DriverHealth.LastSuccessfulRead"/> is the later of
|
||||
/// the two paths' last successes: while either path still reads, the comms-loss watchdog stays quiet; when both
|
||||
/// have stopped it is frozen. Faulted / Unknown / Initializing come from the value (lifecycle) path unchanged.
|
||||
/// </remarks>
|
||||
public DriverHealth GetHealth()
|
||||
{
|
||||
var value = Volatile.Read(ref _health);
|
||||
var ticks = Interlocked.Read(ref _alarmLastSuccessTicks);
|
||||
DateTime? alarmLast = ticks == 0 ? null : new DateTime(ticks, DateTimeKind.Utc);
|
||||
var last = value.LastSuccessfulRead is { } v && (alarmLast is null || v >= alarmLast) ? v : alarmLast;
|
||||
|
||||
if (value.State is not (DriverState.Healthy or DriverState.Degraded)) return value;
|
||||
if (value.State == DriverState.Healthy && !_alarmTransportFailures.IsEmpty)
|
||||
{
|
||||
var reason = _alarmTransportFailures.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase).First().Value;
|
||||
return new DriverHealth(DriverState.Degraded, last, reason);
|
||||
}
|
||||
return value.LastSuccessfulRead == last ? value : value with { LastSuccessfulRead = last };
|
||||
}
|
||||
|
||||
/// <summary>Gets the number of configured devices.</summary>
|
||||
internal int DeviceCount => _devices.Count;
|
||||
@@ -1700,31 +1722,50 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// A successful alarm read is a successful read (Gitea #647 review): it advances
|
||||
/// <see cref="DriverHealth.LastSuccessfulRead"/> exactly as a Good value read does, so an alarm-only driver's
|
||||
/// health tracks its CNC. Never lifts a <see cref="DriverState.Faulted"/> (a config-level signal).
|
||||
/// The ALARM-path half of driver health (Gitea #647 re-review R3): device host → why its alarm read is failing
|
||||
/// at the TRANSPORT. Kept apart from <see cref="_health"/> (the value path) and combined with it in
|
||||
/// <see cref="GetHealth"/>, so neither path can overwrite the other: a good value read cannot mask a failing
|
||||
/// alarm read (and flip /hosts back and forth every poll), and a good alarm read cannot mask a value-path
|
||||
/// Degraded. A CNC refusal never lands here (R2 — see <see cref="FocasAlarmProjection.IsCncRefusal"/>).
|
||||
/// </summary>
|
||||
internal void NoteAlarmReadSucceeded()
|
||||
private readonly ConcurrentDictionary<string, string> _alarmTransportFailures = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>UTC ticks of the last successful alarm read (0 = none) — the alarm path's LastSuccessfulRead.</summary>
|
||||
private long _alarmLastSuccessTicks;
|
||||
|
||||
/// <summary>
|
||||
/// A successful alarm read of <paramref name="hostAddress"/>: clears that device's alarm-path failure and
|
||||
/// advances the alarm path's LastSuccessfulRead, so an alarm-only driver's health tracks its CNC.
|
||||
/// </summary>
|
||||
/// <param name="hostAddress">The device read.</param>
|
||||
internal void NoteAlarmReadSucceeded(string hostAddress)
|
||||
{
|
||||
var current = Volatile.Read(ref _health);
|
||||
if (current.State != DriverState.Faulted)
|
||||
Volatile.Write(ref _health, new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null));
|
||||
_alarmTransportFailures.TryRemove(hostAddress, out _);
|
||||
Interlocked.Exchange(ref _alarmLastSuccessTicks, DateTime.UtcNow.Ticks);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A failed alarm read degrades driver health, PRESERVING <see cref="DriverHealth.LastSuccessfulRead"/> — the
|
||||
/// conjunction the comms-loss watchdog (#527) trips on. Without it a driver whose only tags are <c>ALARM:</c>
|
||||
/// tags stayed Healthy through a total outage, the watchdog never fired, and the host's #477
|
||||
/// <c>BadNoCommunication</c> annotation never ran (Gitea #647 review).
|
||||
/// The CNC REFUSED the alarm read (R2): the transport is fine, so any alarm-path transport failure of the
|
||||
/// device is over — but nothing was read, so LastSuccessfulRead does not advance.
|
||||
/// </summary>
|
||||
/// <param name="hostAddress">The device that answered.</param>
|
||||
internal void NoteAlarmReadRefused(string hostAddress) => _alarmTransportFailures.TryRemove(hostAddress, out _);
|
||||
|
||||
/// <summary>
|
||||
/// A TRANSPORT failure of an alarm read: the alarm path is Degraded until the device's next good read, and its
|
||||
/// LastSuccessfulRead stays frozen — the conjunction the comms-loss watchdog (#527) trips on, which a driver
|
||||
/// whose only tags are <c>ALARM:</c> tags otherwise never reached (Gitea #647 review).
|
||||
/// </summary>
|
||||
/// <param name="hostAddress">The device whose alarm read failed.</param>
|
||||
/// <param name="error">Why.</param>
|
||||
internal void NoteAlarmReadFailed(string hostAddress, Exception error)
|
||||
internal void NoteAlarmReadFailed(string hostAddress, Exception error) =>
|
||||
_alarmTransportFailures[hostAddress] = $"FOCAS alarm read failed for {hostAddress}: {error.Message}";
|
||||
|
||||
/// <summary>Clears the alarm-path health (a fresh device set at initialise / shutdown).</summary>
|
||||
private void ResetAlarmHealth()
|
||||
{
|
||||
var current = Volatile.Read(ref _health);
|
||||
if (current.State != DriverState.Faulted)
|
||||
Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead,
|
||||
$"FOCAS alarm read failed for {hostAddress}: {error.Message}"));
|
||||
_alarmTransportFailures.Clear();
|
||||
Interlocked.Exchange(ref _alarmLastSuccessTicks, 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -119,13 +119,16 @@ public sealed class WireFocasClient : IFocasClient
|
||||
/// Gitea #647: every failure THROWS — a disconnected session, a non-OK return code and a wire fault
|
||||
/// alike. This used to answer <c>[]</c> for all three, which the alarm projection could not tell
|
||||
/// from "no alarms active": every comms blip cleared every active alarm, and recovery re-raised them
|
||||
/// un-acknowledged. <see cref="FocasWireException"/> propagates as-is.
|
||||
/// un-acknowledged. <see cref="FocasWireException"/> propagates as-is; a refusal (non-OK rc) is thrown carrying its <c>Rc</c>.
|
||||
/// </remarks>
|
||||
public async Task<IReadOnlyList<FocasActiveAlarm>> ReadAlarmsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
RequireConnected();
|
||||
var result = await _wire.ReadAlarmsAsync(FocasAlarmType.All, MaxAlarmsPerRead, cancellationToken).ConfigureAwait(false);
|
||||
ThrowIfRcNonZero(result.Rc, "cnc_rdalmmsg2", result.IsOk);
|
||||
// A non-OK return code is the CNC REFUSING the call over a healthy transport — thrown carrying the rc, so
|
||||
// the alarm projection can tell it from comms loss and keep it out of driver health (#647 re-review R2).
|
||||
if (!result.IsOk)
|
||||
throw new FocasWireException($"cnc_rdalmmsg2 refused by the CNC (EW_{result.Rc}).", result.Rc);
|
||||
return (result.Value ?? []).Select(Map).ToList();
|
||||
|
||||
static FocasActiveAlarm Map(WireAlarm a) => new(
|
||||
|
||||
@@ -12,7 +12,7 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
/// </summary>
|
||||
public sealed class NativeAlarmProjector
|
||||
{
|
||||
private readonly Dictionary<string, (bool Active, bool Acked)> _prior = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, (bool Active, bool Acked, string? Message)> _prior = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Project an alarm transition onto the full condition snapshot for <paramref name="nodeId"/>.</summary>
|
||||
/// <param name="nodeId">The materialised condition node's id (the projection's state key).</param>
|
||||
@@ -22,7 +22,8 @@ public sealed class NativeAlarmProjector
|
||||
|
||||
/// <summary>
|
||||
/// Project an alarm transition, reporting whether it was a <b>re-statement</b> (Gitea #647 review): a
|
||||
/// <see cref="AlarmTransitionKind.Raise"/> on a condition already projected Active. That is what a driver
|
||||
/// <see cref="AlarmTransitionKind.Raise"/> on a condition already projected Active WITH THE SAME MESSAGE
|
||||
/// (re-review R1 — a different message is a different alarm, and un-acks). That is what a driver
|
||||
/// whose told-record died with its instance sends after a deploy respawns it — the alarm it re-reads is the
|
||||
/// one the server already shows, so the acknowledgement is KEPT and the host writes no second "Activated"
|
||||
/// row. Only <see cref="AlarmTransitionKind.Retrigger"/> (or inactive → active) un-acknowledges; a source
|
||||
@@ -34,19 +35,25 @@ public sealed class NativeAlarmProjector
|
||||
/// <returns>The full Part 9 condition snapshot to write to the node.</returns>
|
||||
public AlarmConditionSnapshot Project(string nodeId, AlarmEventArgs e, out bool restatement)
|
||||
{
|
||||
var prev = _prior.TryGetValue(nodeId, out var p) ? p : (Active: false, Acked: true);
|
||||
var prev = _prior.TryGetValue(nodeId, out var p) ? p : (Active: false, Acked: true, Message: (string?)null);
|
||||
// #626: a source with a finer scale than the four buckets supplies the exact Part 9 severity.
|
||||
var sev = e.OpcUaSeverity is { } exact ? (ushort)Math.Clamp(exact, 1, 1000) : MapSeverity(e.Severity);
|
||||
restatement = e.Kind == AlarmTransitionKind.Raise && prev.Active;
|
||||
// Gitea #647 re-review R1: only a Raise saying the SAME thing the condition already shows is a re-statement.
|
||||
// A respawned driver cannot tell "the alarm I was showing" from "a different one that appeared while I was
|
||||
// down", so a different message is treated as a new statement — un-acked, with its alerts row. Comparing the
|
||||
// message is conservative: any difference (a sources' messages are deterministic for an unchanged statement)
|
||||
// errs toward showing the operator an unacknowledged alarm, never toward hiding one behind a stale ack.
|
||||
restatement = e.Kind == AlarmTransitionKind.Raise && prev.Active
|
||||
&& string.Equals(prev.Message?.Trim(), e.Message.Trim(), StringComparison.Ordinal);
|
||||
var (active, acked) = e.Kind switch
|
||||
{
|
||||
AlarmTransitionKind.Raise when prev.Active => (true, prev.Acked),
|
||||
AlarmTransitionKind.Raise when restatement => (true, prev.Acked),
|
||||
AlarmTransitionKind.Raise or AlarmTransitionKind.Retrigger => (true, false),
|
||||
AlarmTransitionKind.Acknowledge => (prev.Active, true),
|
||||
AlarmTransitionKind.Clear => (false, prev.Acked),
|
||||
_ => (prev.Active, prev.Acked),
|
||||
};
|
||||
_prior[nodeId] = (active, acked);
|
||||
_prior[nodeId] = (active, acked, e.Message);
|
||||
return new AlarmConditionSnapshot(
|
||||
Active: active, Acknowledged: acked, Confirmed: true, Enabled: true,
|
||||
Shelving: AlarmShelvingKind.Unshelved, Severity: sev, Message: e.Message,
|
||||
@@ -64,7 +71,7 @@ public sealed class NativeAlarmProjector
|
||||
/// <param name="nodeId">The acknowledged condition node id.</param>
|
||||
public void MarkAcknowledged(string nodeId)
|
||||
{
|
||||
if (_prior.TryGetValue(nodeId, out var p)) _prior[nodeId] = (p.Active, true);
|
||||
if (_prior.TryGetValue(nodeId, out var p)) _prior[nodeId] = (p.Active, true, p.Message);
|
||||
}
|
||||
|
||||
/// <summary>Clears all tracked per-node state.</summary>
|
||||
|
||||
Reference in New Issue
Block a user