using System.Collections.Concurrent;
using System.Collections.Frozen;
using System.Globalization;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.RegularExpressions;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
///
/// The driver's live dataItemId → map: the single place
/// an Agent observation's weakly-typed text becomes an OPC UA-shaped value + quality +
/// timestamp. Written by the /current baseline read and by the /sample long-poll
/// pump; read by IReadable.ReadAsync and the subscription fan-out.
///
///
///
/// Quality mapping (design §3.3). MTConnect has exactly one way to say "I have no
/// value": the literal token UNAVAILABLE. It arrives as a Sample/Event's text, and —
/// after normalizes the <Unavailable/>
/// element name — as a CONDITION's value on that same exact spelling. It maps to
/// BadNoCommunication with a null value: the Agent is reachable, the machine's
/// data item is not. The match is ordinal and case-sensitive against the one spelling
/// the parser guarantees; a case-insensitive match would swallow a legitimate free-text
/// EVENT whose value happens to read "Unavailable".
///
///
/// Failure posture. This type sits under IReadable and must never throw across
/// that boundary on account of observation content. Every unusable observation degrades to a
/// Bad-coded snapshot that still carries the Agent's source timestamp, so an operator can
/// always see when the driver last heard about the tag even when it cannot show
/// what. The only exceptions raised are for a null
/// argument — a programming error, not data.
///
///
/// Status codes are the canonical Opc.Ua.StatusCodes numeric values, declared
/// once each below. Note that is 0x80740000: the
/// 0x80730000 that four sibling drivers declare under that name is really
/// BadWriteNotSupported, which would be actively misleading on a read path and
/// renders as bare "Bad" in the driver CLI's SnapshotFormatter.
///
///
/// Thread safety. Backed by a and
/// guaranteed per-key atomic — nothing more, deliberately. writes
/// each dataItemId exactly once per call with a single whole-snapshot assignment, so a
/// reader can never observe a torn snapshot (a Good code beside a stale value). It is
/// explicitly not a whole-batch atomic swap: OPC UA reads are per-tag and the driver
/// offers no cross-tag consistency contract, /sample chunks are deltas whose
/// interleaving is indistinguishable from two adjacent legal chunks, and a whole-index lock
/// would serialize the pump against every read for no semantic gain.
///
///
/// Shapes this build does not represent are coded BadNotSupported with a null
/// value rather than approximated: a non-Float64 array tag and a DATA_SET / TABLE
/// observation whose children are not <Entry> elements (a Float64 TIME_SERIES vector is served as double[], #627). The latter carries its
/// real content in <Entry key=…> child elements, which reach a text-valued
/// observation concatenated with the keys discarded — a two-entry set reads as the single
/// token "12" — and because the type inference correctly demotes those
/// representations to , coercion would otherwise
/// "succeed" into a Good-coded snapshot carrying noise. This index cannot detect that shape
/// itself (no value heuristic can separate concatenated entries from a legitimate
/// space-bearing EVENT such as a Message reading "Coolant level low"), so it relies on
/// , which
/// sets while the XML is still XML.
///
///
internal sealed class MTConnectObservationIndex
{
///
/// The one token that means "the Agent has no value for this data item". Matched ordinally:
/// the parser already normalizes a CONDITION's <Unavailable/> element name onto
/// this exact spelling, so every category lands on one comparison.
///
private const string UnavailableSentinel = "UNAVAILABLE";
private const uint Good = 0x00000000u;
/// The Agent reported UNAVAILABLE, or supplied no text at all.
private const uint BadNoCommunication = 0x80310000u;
/// The tag is authored but the Agent has not yet reported it.
private const uint BadWaitingForInitialData = 0x80320000u;
/// The dataItemId is neither authored nor ever observed.
private const uint BadNodeIdUnknown = 0x80340000u;
/// The Agent reported a number the authored type cannot represent.
private const uint BadOutOfRange = 0x803C0000u;
/// The observation's shape is real data this build cannot represent (a DATA_SET/TABLE, or a non-Float64 array tag).
private const uint BadNotSupported = 0x803D0000u;
/// A TIME_SERIES vector whose element count disagrees with its declared sampleCount.
private const uint BadDecodingError = 0x80070000u;
/// The Agent reported text that is not a value of the authored type at all.
private const uint BadTypeMismatch = 0x80740000u;
///
/// MTConnect's CONDITION state vocabulary, ranked by severity. Used only to reconcile
/// several states reported for one dataItemId within a single document — see
/// . An active Fault outranks UNAVAILABLE because a
/// fault is information and an absent value is not. Case-insensitive so a vendor's casing
/// of a state element still ranks; the parser itself emits the canonical spellings.
///
private static readonly FrozenDictionary ConditionSeverity =
new Dictionary(StringComparer.OrdinalIgnoreCase)
{
[UnavailableSentinel] = 0,
["Normal"] = 1,
["Warning"] = 2,
["Fault"] = 3,
}.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary _snapshots = new(StringComparer.Ordinal);
// #650: the accumulated DATA_SET / TABLE content per dataItemId. A /sample chunk carries only the entries
// that changed, so the served JSON is the merge of the baseline and every delta since. Row value is the
// entry's text (DATA_SET) or its cell map (TABLE). Only the single pump (or a one-shot snapshot) calls
// Apply, but the lock keeps a concurrent one-shot read-through from corrupting a shared index.
private readonly Dictionary> _sets = new(StringComparer.Ordinal);
private readonly object _setsLock = new();
private bool _anySets;
private sealed record SetEntry(string? Value, SortedDictionary? Cells);
private static readonly Regex JsonNumber = new(
@"^-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
private readonly FrozenDictionary _tags;
private readonly TimeProvider _timeProvider;
///
/// Builds an index over the driver instance's authored tags.
///
///
/// The authored tags, keyed by (the DataItem
/// id) — normally . null or empty is
/// legal and means "no authored types": every observation is then indexed as its raw text
/// under .
///
/// Nothing enforces FullName uniqueness at the options layer, so a duplicate is
/// operator-authorable. The last definition wins rather than throwing: refusing to
/// construct would turn an authoring slip into a driver that will not start, whereas
/// last-wins is deterministic and costs only that one tag's coercion type.
///
///
/// Clock behind ServerTimestampUtc; defaults to .
public MTConnectObservationIndex(
IEnumerable? tags = null,
TimeProvider? timeProvider = null)
{
_timeProvider = timeProvider ?? TimeProvider.System;
var map = new Dictionary(StringComparer.Ordinal);
foreach (var tag in tags ?? [])
{
if (tag is null || string.IsNullOrWhiteSpace(tag.FullName)) continue;
map[tag.FullName] = tag;
}
_tags = map.ToFrozenDictionary(StringComparer.Ordinal);
}
/// The number of distinct dataItemIds currently indexed.
public int Count => _snapshots.Count;
///
/// Folds a /current snapshot or one /sample chunk into the index. Observations
/// with no authored tag are indexed too — the Agent streams the whole device, so they are
/// normal, and dropping them would make a streaming-but-not-yet-authored data item
/// indistinguishable from one that does not exist.
///
/// The parsed streams document. An observation-free result is a legal
/// keep-alive and leaves the index untouched.
/// is null.
///
/// true for a /current document, which states the whole DATA_SET / TABLE and so replaces
/// the accumulated set; false (default) for a /sample chunk, whose entries are deltas merged into it.
///
public void Apply(MTConnectStreamsResult result, bool baseline = false)
{
ArgumentNullException.ThrowIfNull(result);
if (result.Observations is not { Count: > 0 }) return;
// Duplicates are reconciled on the RAW observations first, so each dataItemId is written
// exactly once below as a single atomic whole-snapshot assignment. Doing it the other way
// — writing every observation and merging snapshots in place — would need a read-modify-write
// per key and could expose a half-merged state to a concurrent reader.
var resolved = new Dictionary(StringComparer.Ordinal);
var seenSets = new HashSet(StringComparer.Ordinal);
foreach (var raw in result.Observations)
{
if (raw is null || string.IsNullOrWhiteSpace(raw.DataItemId)) continue;
var observation = FoldDataSet(raw, baseline, seenSets);
resolved[observation.DataItemId] =
resolved.TryGetValue(observation.DataItemId, out var incumbent)
? Reconcile(incumbent, observation)
: observation;
}
foreach (var (dataItemId, observation) in resolved)
{
_snapshots[dataItemId] = BuildSnapshot(observation);
}
}
///
/// Returns the current snapshot for a DataItem. Never null and never throws: an
/// unknown or not-yet-reported id yields a Bad-coded snapshot rather than an error.
///
/// The DataItem id the tag binds.
///
/// The indexed snapshot; else BadWaitingForInitialData when the id is authored but
/// the Agent has not reported it yet (the standard OPC UA "subscribed, no value yet"
/// state); else BadNodeIdUnknown. The two are kept distinct because collapsing them
/// would tell an operator their correctly-authored tag does not exist.
///
public DataValueSnapshot Get(string dataItemId)
{
if (!string.IsNullOrWhiteSpace(dataItemId) &&
_snapshots.TryGetValue(dataItemId, out var snapshot))
{
return snapshot;
}
var status = !string.IsNullOrWhiteSpace(dataItemId) && _tags.ContainsKey(dataItemId)
? BadWaitingForInitialData
: BadNodeIdUnknown;
return new DataValueSnapshot(null, status, null, Now);
}
///
/// Drops every indexed observation. The Agent-restart re-baseline: when the streams header's
/// instanceId changes, every value the index holds describes a device model that no
/// longer exists, and serving it would be worse than serving nothing.
///
public void Clear()
{
_snapshots.Clear();
lock (_setsLock) _sets.Clear();
}
///
/// DATA_SET / TABLE (#650). Folds an observation's <Entry> children into the accumulated set for
/// its dataItemId and returns a plain String observation carrying the merged JSON; any other observation is
/// returned untouched, except that an UNAVAILABLE one drops the accumulated set (a later delta must not
/// resurrect pre-outage entries).
///
/// Semantics (MTConnect 2.x): a /current document () or an observation
/// with resetTriggered replaces the set; otherwise entries merge by key. removed="true"
/// deletes the key (the key is absent from the JSON, never null). A TABLE row is replaced as a unit
/// by a later entry for the same row key; a removed row key disappears. Several observations of one item in
/// a single document fold in document order — only the first replaces on a baseline.
///
///
private MTConnectObservation FoldDataSet(MTConnectObservation observation, bool baseline, HashSet seenSets)
{
var id = observation.DataItemId;
if (observation.Entries is not { Count: > 0 } entries)
{
var unavailable = string.IsNullOrWhiteSpace(observation.Value) && !observation.IsDataSet ||
string.Equals(observation.Value, UnavailableSentinel, StringComparison.Ordinal);
if (unavailable)
{
// Hot path for every ordinary UNAVAILABLE observation: skip the lock until a set exists at all.
if (Volatile.Read(ref _anySets)) lock (_setsLock) _sets.Remove(id);
return observation;
}
if (observation.IsDataSet && !observation.IsStructured)
{
// An entry-less data-set-shaped observation (count="0", or resetTriggered) is the EMPTY set - a
// legitimate value, Good "{}", not an absence. A reset or a /current baseline discards what
// was accumulated; a bare empty delta leaves the set as it is.
string emptyJson;
lock (_setsLock)
{
var first = seenSets.Add(id);
if (!_sets.TryGetValue(id, out var set) || observation.ResetTriggered || (baseline && first))
{
set = new SortedDictionary(StringComparer.Ordinal);
_sets[id] = set;
Volatile.Write(ref _anySets, true);
}
emptyJson = RenderSet(set);
}
return observation with { Value = emptyJson, ResetTriggered = false, IsDataSet = false };
}
return observation;
}
string json;
lock (_setsLock)
{
var first = seenSets.Add(id);
if (!_sets.TryGetValue(id, out var set) || observation.ResetTriggered || (baseline && first))
{
set = new SortedDictionary(StringComparer.Ordinal);
_sets[id] = set;
Volatile.Write(ref _anySets, true);
}
foreach (var entry in entries)
{
if (entry.Removed)
{
set.Remove(entry.Key);
}
else if (entry.Cells is not null)
{
var cells = new SortedDictionary(StringComparer.Ordinal);
foreach (var cell in entry.Cells) cells[cell.Key] = cell.Value;
set[entry.Key] = new SetEntry(null, cells);
}
else
{
set[entry.Key] = new SetEntry(entry.Value ?? string.Empty, null);
}
}
json = RenderSet(set);
}
return observation with { Value = json, IsStructured = false, Entries = null, ResetTriggered = false, IsDataSet = false };
}
///
/// Compact, deterministic JSON: keys ordinal-sorted, no whitespace. Values that are strict JSON numbers are
/// emitted as numbers with their original token (lossless — 007 and +5 stay strings); everything
/// else is a string.
///
private static string RenderSet(SortedDictionary set)
{
using var stream = new MemoryStream();
using (var w = new Utf8JsonWriter(stream, new JsonWriterOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }))
{
w.WriteStartObject();
foreach (var (key, entry) in set)
{
w.WritePropertyName(key);
if (entry.Cells is null)
{
WriteScalar(w, entry.Value ?? string.Empty);
continue;
}
w.WriteStartObject();
foreach (var (cellKey, cellValue) in entry.Cells)
{
w.WritePropertyName(cellKey);
WriteScalar(w, cellValue);
}
w.WriteEndObject();
}
w.WriteEndObject();
}
return System.Text.Encoding.UTF8.GetString(stream.ToArray());
}
private static void WriteScalar(Utf8JsonWriter w, string text)
{
if (JsonNumber.IsMatch(text) &&
double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var d) && double.IsFinite(d))
{
w.WriteRawValue(text);
}
else
{
w.WriteStringValue(text);
}
}
private DateTime Now => _timeProvider.GetUtcNow().UtcDateTime;
///
/// Picks the winner when one dataItemId appears more than once in a single document. A
/// <Condition> container legitimately carries several simultaneously active
/// states (a Fault and a Warning at once), so for a pair of recognized condition state words
/// the most severe wins — plain last-wins would under-report an active Fault as a
/// Warning purely because of element order. Anything else is last-wins, matching the Agent's
/// ascending-sequence ordering.
///
/// Deliberately scoped to one document: a later is the Agent's new
/// statement of the state, so a cleared fault must fall back to Normal. A worst-of that
/// spanned Applies would latch every fault forever.
///
///
private static MTConnectObservation Reconcile(MTConnectObservation incumbent, MTConnectObservation challenger)
{
if (ConditionSeverity.TryGetValue(incumbent.Value ?? string.Empty, out var incumbentSeverity) &&
ConditionSeverity.TryGetValue(challenger.Value ?? string.Empty, out var challengerSeverity))
{
return challengerSeverity >= incumbentSeverity ? challenger : incumbent;
}
return challenger;
}
private DataValueSnapshot BuildSnapshot(MTConnectObservation observation)
{
var serverTimestamp = Now;
var sourceTimestamp = DateTime.SpecifyKind(observation.TimestampUtc, DateTimeKind.Utc);
// No text at all is the degenerate spelling of "no value": MTConnect defines UNAVAILABLE as
// the only way to report absence, so an empty element is not an empty string value.
if (string.IsNullOrWhiteSpace(observation.Value) ||
string.Equals(observation.Value, UnavailableSentinel, StringComparison.Ordinal))
{
return new DataValueSnapshot(null, BadNoCommunication, sourceTimestamp, serverTimestamp);
}
// DATA_SET / TABLE: the Agent carried the content in child elements, so
// Value is their concatenated text with the keys discarded (a two-entry set reads "12").
// Publishing that as a Good string would be noise an operator would trust. Checked before
// the tag lookup because it is a fact about the wire shape, true whether or not the data
// item is authored — and after the sentinel, so an unavailable data set still reports the
// truthful no-comms.
if (observation.IsStructured)
{
return new DataValueSnapshot(null, BadNotSupported, sourceTimestamp, serverTimestamp);
}
_tags.TryGetValue(observation.DataItemId, out var tag);
// TIME_SERIES vectors arrive as one whitespace-separated string (#627). Checked AFTER the
// sentinel so an unavailable array tag still reports the truthful no-comms.
if (tag is { IsArray: true })
{
return BuildArraySnapshot(observation, tag, sourceTimestamp, serverTimestamp);
}
// An unauthored observation has no declared target type; String is the only coercion that
// cannot fail, and it carries the Agent's text with no interpretation applied.
var status = Coerce(observation.Value, tag?.DriverDataType ?? DriverDataType.String, out var value);
return status == Good
? new DataValueSnapshot(value, Good, sourceTimestamp, serverTimestamp)
: new DataValueSnapshot(null, status, sourceTimestamp, serverTimestamp);
}
///
/// Parses a TIME_SERIES vector into a double[]. Only
/// arrays are served (the MTConnect TIME_SERIES element type is a floating-point sample); any other
/// authored array type is BadNotSupported rather than an approximation. All-or-nothing: one
/// unparseable or non-finite token, or a count that disagrees with the observation's own
/// sampleCount (or the tag's fixed ArrayDim), makes the whole vector Bad — a partial
/// vector served as Good would be a silently wrong waveform.
///
private static DataValueSnapshot BuildArraySnapshot(
MTConnectObservation observation, MTConnectTagDefinition tag, DateTime sourceTimestamp, DateTime serverTimestamp)
{
DataValueSnapshot Bad(uint status) => new(null, status, sourceTimestamp, serverTimestamp);
if (tag.DriverDataType != DriverDataType.Float64) return Bad(BadNotSupported);
var tokens = observation.Value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);
var values = new double[tokens.Length];
for (var i = 0; i < tokens.Length; i++)
{
if (string.Equals(tokens[i], UnavailableSentinel, StringComparison.Ordinal)) return Bad(BadNoCommunication);
if (!double.TryParse(tokens[i], NumberStyles.Float, CultureInfo.InvariantCulture, out values[i]) ||
!double.IsFinite(values[i]))
{
return Bad(BadTypeMismatch);
}
}
if (observation.SampleCount is { } declared && declared != values.Length) return Bad(BadDecodingError);
if (tag.ArrayDim > 0 && tag.ArrayDim != values.Length) return Bad(BadDecodingError);
return new DataValueSnapshot(values, Good, sourceTimestamp, serverTimestamp);
}
///
/// Converts the Agent's text to the tag's authored type. Every parse is
/// : MTConnect is an invariant-format wire, and a
/// host running a comma-decimal culture would otherwise read 123.4567 as
/// 1234567 — a silently wrong value under Good quality, the worst possible outcome.
///
/// on success; otherwise a Bad code and a null value.
private static uint Coerce(string raw, DriverDataType type, out object? value)
{
value = null;
var text = raw.Trim();
switch (type)
{
// Reference is a Galaxy-style attribute reference encoded as an OPC UA String. It is
// never a sensible authoring for MTConnect, but degrading it to text is harmless where
// rejecting it would break a tag for no protection.
case DriverDataType.String:
case DriverDataType.Reference:
value = raw;
return Good;
case DriverDataType.Boolean:
if (bool.TryParse(text, out var flag)) { value = flag; return Good; }
if (text is "1") { value = true; return Good; }
if (text is "0") { value = false; return Good; }
return BadTypeMismatch;
case DriverDataType.DateTime:
if (DateTime.TryParse(
text,
CultureInfo.InvariantCulture,
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal,
out var moment))
{
value = DateTime.SpecifyKind(moment, DateTimeKind.Utc);
return Good;
}
return BadTypeMismatch;
case DriverDataType.Float32:
if (float.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var single))
{
value = single;
return Good;
}
return BadTypeMismatch;
case DriverDataType.Float64:
if (double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var real))
{
value = real;
return Good;
}
return BadTypeMismatch;
case DriverDataType.Int16:
if (short.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var i16))
{
value = i16;
return Good;
}
break;
case DriverDataType.Int32:
if (int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var i32))
{
value = i32;
return Good;
}
break;
case DriverDataType.Int64:
if (long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var i64))
{
value = i64;
return Good;
}
break;
case DriverDataType.UInt16:
if (ushort.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var u16))
{
value = u16;
return Good;
}
break;
case DriverDataType.UInt32:
if (uint.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var u32))
{
value = u32;
return Good;
}
break;
case DriverDataType.UInt64:
if (ulong.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var u64))
{
value = u64;
return Good;
}
break;
default:
// A DriverDataType this driver has no rule for. Text always round-trips, so serving
// it beats failing a tag over an enum member added later.
value = raw;
return Good;
}
// Integer parse failed. Distinguishing the two causes is worth one extra parse: it tells an
// operator whether to retype the tag or widen it.
return double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out _)
? BadOutOfRange // a number the authored type cannot represent (overflow, or a fraction)
: BadTypeMismatch; // not a number at all
}
}