dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(alarms): #647 re-review — re-statement needs the same message; CNC refusal is not comms loss; worst-of health

Browse Source 
R1: NativeAlarmProjector treats Raise-on-Active as a re-statement only when the message matches what
the condition shows; a different alarm appearing during a respawn un-acks and writes its alerts row.
FOCAS condition messages are a deterministic function of the matched set (tested across fresh driver
instances and reordered lists); Runtime E2E now covers both the same alarm and a new one during a
respawn. Residual (source message determinism) documented in AlarmTracking.md.

R2: a non-OK cnc_rdalmmsg2 return is thrown as FocasWireException{Rc} and classified as a CNC
refusal: conditions get a quality-only BadNotSupported once, one Warning, polling continues, and
driver health is untouched (no 90 s watchdog reconnect loop on a production control). Only transport
faults degrade (falsification control in the test).

R3: alarm-path health is tracked apart from the value path and combined in GetHealth: worst state,
later LastSuccessfulRead. A good value read can no longer mask a failing alarm read (no per-poll
/hosts flip) and a good alarm read cannot mask a value Degraded.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-08 20:45:32 -04:00
parent 89961af6e6
commit bac93744ce
11 changed files with 308 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasAlarmProjection.cs
+48 -18
View File
@@ -57,22 +57,37 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
/// <summary>Devices whose last read came back at the per-read cap. Guarded by <see cref="_alarmLock"/>.</summary>
private readonly HashSet<string> _truncationWarned = new(StringComparer.OrdinalIgnoreCase);
/// <summary>Devices whose alarm read is currently failing (one Warning per episode). Guarded by <see cref="_alarmLock"/>.</summary>
private readonly HashSet<string> _failingDevices = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Devices whose alarm read is currently failing → the quality that failure maps to (one Warning per episode
/// and per kind: a refusal turning into a transport fault warns again). Guarded by <see cref="_alarmLock"/>.
/// </summary>
private readonly Dictionary<string, uint> _failingDevices = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// ConditionIds this projection has annotated <c>BadNoCommunication</c> and not yet lifted. Guarded by
/// <see cref="_alarmLock"/>. Kept apart from <see cref="_told"/>: a quality annotation never touches a
/// condition's Active/Acked state.
/// </summary>
private readonly HashSet<string> _qualityBadTold = new(StringComparer.Ordinal);
private readonly Dictionary<string, uint> _qualityBadTold = new(StringComparer.Ordinal);
/// <summary>Canonical <c>Opc.Ua.StatusCodes.Good</c>.</summary>
private const uint QualityGood = 0x00000000u;
/// <summary>Canonical <c>Opc.Ua.StatusCodes.BadNoCommunication</c>.</summary>
/// <summary>Canonical <c>Opc.Ua.StatusCodes.BadNoCommunication</c> — the CNC could not be reached.</summary>
private const uint QualityBadNoCommunication = 0x80310000u;
/// <summary>
/// Whether an alarm-read failure is a CNC REFUSAL (Gitea #647 re-review R2): the CNC answered with a non-OK
/// FOCAS return code, so the transport is fine and the control simply will not serve <c>cnc_rdalmmsg2</c>
/// (e.g. <c>EW_FUNC</c>, as <c>cnc_rdparam</c> does on the Makino). That is a capability answer, not comms
/// loss: it must NOT degrade driver health, or an alarm-only driver on such a control would be reconnected by
/// the #527 watchdog every 90 s forever. Everything else (session down, wire fault, malformed reply, backoff)
/// is a transport failure.
/// </summary>
/// <param name="error">The alarm-read failure.</param>
/// <returns><see langword="true"/> for a refusal.</returns>
internal static bool IsCncRefusal(Exception error) => error is Wire.FocasWireException { Rc: not null };
/// <summary>Serializes loop start/stop.</summary>
private readonly SemaphoreSlim _loopGate = new(1, 1);
@@ -246,13 +261,15 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
catch (Exception ex)
{
// Not "no alarms": keep every condition on this device exactly as the server was last told, but say
// its source is unreachable — and let the failure reach driver health (an alarm-only driver has no
// other read to trip the comms-loss watchdog with).
_driver.NoteAlarmReadFailed(host, ex);
// why it cannot be read. A transport failure reaches driver health (an alarm-only driver has no other
// read to trip the comms-loss watchdog with); a CNC refusal does not (R2). Polling continues at the
// normal cadence either way.
if (!IsCncRefusal(ex)) _driver.NoteAlarmReadFailed(host, ex);
else _driver.NoteAlarmReadRefused(host); // the CNC answered: the transport to it is fine
TickFailed(sub, host, tags, ex);
continue;
}
_driver.NoteAlarmReadSucceeded();
_driver.NoteAlarmReadSucceeded(host);
Tick(sub, host, alarms, tags);
}
}
@@ -290,14 +307,14 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
var handlers = _driver.AlarmHandlers;
if (handlers is null) return;
if (_failingDevices.Remove(deviceHostAddress))
if (_failingDevices.Remove(deviceHostAddress, out _))
_logger.LogInformation("FOCAS alarm read for {Host} recovered", deviceHostAddress);
foreach (var tag in deviceTags.Where(t => sub.References.Contains(t.RawPath)))
{
// Lift a Bad this projection put there BEFORE the normal diff, so a recovery that also changed the
// alarm state (cleared during the outage) lands as Good then Clear.
if (_qualityBadTold.Remove(tag.RawPath))
if (_qualityBadTold.Remove(tag.RawPath, out _))
Deliver(handlers, QualityOnly(tag, sub.Handle, QualityGood, now), tag.RawPath);
var view = FocasAlarmMatcher.Evaluate(tag.Selector, alarms, deviceSelectors, truncated);
@@ -312,8 +329,9 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
/// <summary>
/// A FAILED read of <paramref name="deviceHostAddress"/> (Gitea #647 review): conditions keep their state, but
/// each subscribed one is annotated <c>BadNoCommunication</c> once per outage through the #626 quality-only
/// path (no Active/Acked change, no <c>/alerts</c> row), and the first failure of the episode logs one
/// each subscribed one is annotated once per outage through the #626 quality-only path (no Active/Acked change,
/// no <c>/alerts</c> row) — <c>BadNoCommunication</c> for a transport failure, <c>BadNotSupported</c> for a CNC
/// refusal (R2, <see cref="IsCncRefusal"/>) — and the first failure of the episode (per kind) logs one
/// Warning. The next successful read lifts the Bad in <see cref="Tick"/>.
/// </summary>
/// <param name="sub">The subscription the read was taken for; a superseded one emits nothing.</param>
@@ -328,10 +346,21 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
{
if (!ReferenceEquals(Volatile.Read(ref _current), sub)) return;
if (_failingDevices.Add(deviceHostAddress))
_logger.LogWarning(error,
"FOCAS alarm read (cnc_rdalmmsg2) failed for {Host}; its alarm conditions keep their state and are " +
"marked BadNoCommunication until a read succeeds", deviceHostAddress);
var refusal = IsCncRefusal(error);
var quality = refusal ? FocasStatusMapper.BadNotSupported : QualityBadNoCommunication;
if (!_failingDevices.TryGetValue(deviceHostAddress, out var was) || was != quality)
{
_failingDevices[deviceHostAddress] = quality;
if (refusal)
_logger.LogWarning(error,
"FOCAS CNC {Host} refuses the alarm read (cnc_rdalmmsg2); its alarm conditions keep their state " +
"and are marked BadNotSupported. This is a capability answer, not comms loss — driver health is " +
"unaffected and polling continues", deviceHostAddress);
else
_logger.LogWarning(error,
"FOCAS alarm read (cnc_rdalmmsg2) failed for {Host}; its alarm conditions keep their state and " +
"are marked BadNoCommunication until a read succeeds", deviceHostAddress);
}
else
_logger.LogDebug(error, "FOCAS alarm read for {Host} still failing", deviceHostAddress);
@@ -343,8 +372,9 @@ internal sealed class FocasAlarmProjection : IAsyncDisposable
string.Equals(t.DeviceHostAddress, deviceHostAddress, StringComparison.OrdinalIgnoreCase)
&& sub.References.Contains(t.RawPath)))
{
if (!_qualityBadTold.Add(tag.RawPath)) continue;
Deliver(handlers, QualityOnly(tag, sub.Handle, QualityBadNoCommunication, now), tag.RawPath);
if (_qualityBadTold.TryGetValue(tag.RawPath, out var told) && told == quality) continue;
_qualityBadTold[tag.RawPath] = quality;
Deliver(handlers, QualityOnly(tag, sub.Handle, quality, now), tag.RawPath);
}
}
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs
+58 -17
View File
@@ -200,6 +200,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
// AdoptConfigDuringInit calls this without a teardown first). Not ShutdownAsync: that also drops _poll
// and subscriptions, which are owned by the shutdown path (the actor re-subscribes after every successful init).
await ReleaseDeviceStatesAsync().ConfigureAwait(false);
ResetAlarmHealth();
Volatile.Write(ref _health, new DriverHealth(DriverState.Initializing, null, null));
try
{
@@ -475,11 +476,32 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
_resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry
_parsedAddressesByTagName.Clear();
_fixedTreeActive = false;
ResetAlarmHealth();
Volatile.Write(ref _health, new DriverHealth(DriverState.Unknown, Volatile.Read(ref _health).LastSuccessfulRead, null));
}
/// <inheritdoc />
public DriverHealth GetHealth() => Volatile.Read(ref _health);
/// <remarks>
/// The worst of the value path (<see cref="_health"/>) and the alarm path (<see cref="_alarmTransportFailures"/>),
/// computed together (Gitea #647 re-review R3). <see cref="DriverHealth.LastSuccessfulRead"/> is the later of
/// the two paths' last successes: while either path still reads, the comms-loss watchdog stays quiet; when both
/// have stopped it is frozen. Faulted / Unknown / Initializing come from the value (lifecycle) path unchanged.
/// </remarks>
public DriverHealth GetHealth()
{
var value = Volatile.Read(ref _health);
var ticks = Interlocked.Read(ref _alarmLastSuccessTicks);
DateTime? alarmLast = ticks == 0 ? null : new DateTime(ticks, DateTimeKind.Utc);
var last = value.LastSuccessfulRead is { } v && (alarmLast is null || v >= alarmLast) ? v : alarmLast;
if (value.State is not (DriverState.Healthy or DriverState.Degraded)) return value;
if (value.State == DriverState.Healthy && !_alarmTransportFailures.IsEmpty)
{
var reason = _alarmTransportFailures.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase).First().Value;
return new DriverHealth(DriverState.Degraded, last, reason);
}
return value.LastSuccessfulRead == last ? value : value with { LastSuccessfulRead = last };
}
/// <summary>Gets the number of configured devices.</summary>
internal int DeviceCount => _devices.Count;
@@ -1700,31 +1722,50 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
/// <summary>
/// A successful alarm read is a successful read (Gitea #647 review): it advances
/// <see cref="DriverHealth.LastSuccessfulRead"/> exactly as a Good value read does, so an alarm-only driver's
/// health tracks its CNC. Never lifts a <see cref="DriverState.Faulted"/> (a config-level signal).
/// The ALARM-path half of driver health (Gitea #647 re-review R3): device host → why its alarm read is failing
/// at the TRANSPORT. Kept apart from <see cref="_health"/> (the value path) and combined with it in
/// <see cref="GetHealth"/>, so neither path can overwrite the other: a good value read cannot mask a failing
/// alarm read (and flip /hosts back and forth every poll), and a good alarm read cannot mask a value-path
/// Degraded. A CNC refusal never lands here (R2 — see <see cref="FocasAlarmProjection.IsCncRefusal"/>).
/// </summary>
internal void NoteAlarmReadSucceeded()
private readonly ConcurrentDictionary<string, string> _alarmTransportFailures = new(StringComparer.OrdinalIgnoreCase);
/// <summary>UTC ticks of the last successful alarm read (0 = none) — the alarm path's LastSuccessfulRead.</summary>
private long _alarmLastSuccessTicks;
/// <summary>
/// A successful alarm read of <paramref name="hostAddress"/>: clears that device's alarm-path failure and
/// advances the alarm path's LastSuccessfulRead, so an alarm-only driver's health tracks its CNC.
/// </summary>
/// <param name="hostAddress">The device read.</param>
internal void NoteAlarmReadSucceeded(string hostAddress)
{
var current = Volatile.Read(ref _health);
if (current.State != DriverState.Faulted)
Volatile.Write(ref _health, new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null));
_alarmTransportFailures.TryRemove(hostAddress, out _);
Interlocked.Exchange(ref _alarmLastSuccessTicks, DateTime.UtcNow.Ticks);
}
/// <summary>
/// A failed alarm read degrades driver health, PRESERVING <see cref="DriverHealth.LastSuccessfulRead"/> — the
/// conjunction the comms-loss watchdog (#527) trips on. Without it a driver whose only tags are <c>ALARM:</c>
/// tags stayed Healthy through a total outage, the watchdog never fired, and the host's #477
/// <c>BadNoCommunication</c> annotation never ran (Gitea #647 review).
/// The CNC REFUSED the alarm read (R2): the transport is fine, so any alarm-path transport failure of the
/// device is over — but nothing was read, so LastSuccessfulRead does not advance.
/// </summary>
/// <param name="hostAddress">The device that answered.</param>
internal void NoteAlarmReadRefused(string hostAddress) => _alarmTransportFailures.TryRemove(hostAddress, out _);
/// <summary>
/// A TRANSPORT failure of an alarm read: the alarm path is Degraded until the device's next good read, and its
/// LastSuccessfulRead stays frozen — the conjunction the comms-loss watchdog (#527) trips on, which a driver
/// whose only tags are <c>ALARM:</c> tags otherwise never reached (Gitea #647 review).
/// </summary>
/// <param name="hostAddress">The device whose alarm read failed.</param>
/// <param name="error">Why.</param>
internal void NoteAlarmReadFailed(string hostAddress, Exception error)
internal void NoteAlarmReadFailed(string hostAddress, Exception error) =>
_alarmTransportFailures[hostAddress] = $"FOCAS alarm read failed for {hostAddress}: {error.Message}";
/// <summary>Clears the alarm-path health (a fresh device set at initialise / shutdown).</summary>
private void ResetAlarmHealth()
{
var current = Volatile.Read(ref _health);
if (current.State != DriverState.Faulted)
Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead,
$"FOCAS alarm read failed for {hostAddress}: {error.Message}"));
_alarmTransportFailures.Clear();
Interlocked.Exchange(ref _alarmLastSuccessTicks, 0);
}
/// <summary>
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/WireFocasClient.cs
+5 -2
View File
@@ -119,13 +119,16 @@ public sealed class WireFocasClient : IFocasClient
/// Gitea #647: every failure THROWS — a disconnected session, a non-OK return code and a wire fault
/// alike. This used to answer <c>[]</c> for all three, which the alarm projection could not tell
/// from "no alarms active": every comms blip cleared every active alarm, and recovery re-raised them
/// un-acknowledged. <see cref="FocasWireException"/> propagates as-is.
/// un-acknowledged. <see cref="FocasWireException"/> propagates as-is; a refusal (non-OK rc) is thrown carrying its <c>Rc</c>.
/// </remarks>
public async Task<IReadOnlyList<FocasActiveAlarm>> ReadAlarmsAsync(CancellationToken cancellationToken)
{
RequireConnected();
var result = await _wire.ReadAlarmsAsync(FocasAlarmType.All, MaxAlarmsPerRead, cancellationToken).ConfigureAwait(false);
ThrowIfRcNonZero(result.Rc, "cnc_rdalmmsg2", result.IsOk);
// A non-OK return code is the CNC REFUSING the call over a healthy transport — thrown carrying the rc, so
// the alarm projection can tell it from comms loss and keep it out of driver health (#647 re-review R2).
if (!result.IsOk)
throw new FocasWireException($"cnc_rdalmmsg2 refused by the CNC (EW_{result.Rc}).", result.Rc);
return (result.Value ?? []).Select(Map).ToList();
static FocasActiveAlarm Map(WireAlarm a) => new(