Merge b647-mtc-dataset: MTConnect DATA_SET/TABLE as JSON strings; EVENT enumerations parked (#650)

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectDriver.cs

@@ -747,7 +747,7 @@ public sealed partial class MTConnectDriver
             // Built from the SAME authored tags as the shared index, so the authored-vs-unknown
             // distinction and every coercion behave identically — it is a snapshot, not a variant.
             var snapshot = new MTConnectObservationIndex(binding.IndexTags);
-            snapshot.Apply(current);
+            snapshot.Apply(current, baseline: true);
 
             var results = new DataValueSnapshot[fullReferences.Count];
             for (var i = 0; i < results.Length; i++)
@@ -2337,7 +2337,7 @@ public sealed partial class MTConnectDriver
     private void ApplyAndFanOut(MTConnectStreamsResult document, bool baseline = false)
     {
         var index = ObservationIndex;
-        index.Apply(document);
+        index.Apply(document, baseline);
 
         // #626: CONDITION observations also feed the alarm path, which keeps its own multi-condition state
         // (a /sample Normal for one nativeCode is not a return to normal while another code is active).
@@ -2801,7 +2801,7 @@ public sealed partial class MTConnectDriver
             }
 
             var index = new MTConnectObservationIndex(binding.IndexTags);
-            index.Apply(current);
+            index.Apply(current, baseline: true);
             Volatile.Write(ref _index, index);
 
             // #626: the CONDITION tracker is per-session state like the index. Nothing is RAISED here — the

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectDtos.cs

@@ -167,6 +167,23 @@ public sealed record MTConnectStreamsResult(
 ///     observation. Carried for the parser contract; not yet surfaced on the served node (see
 ///     docs/drivers/MTConnect.md). <c>null</c> when absent.
 /// </param>
+/// <param name="Entries">
+///     The parsed <c>&lt;Entry key="…"&gt;</c> children of a <c>DATA_SET</c> / <c>TABLE</c> observation (#650), in
+///     document order, or <c>null</c> when the observation had none. A <c>TABLE</c> entry carries
+///     <see cref="MTConnectDataSetEntry.Cells"/>. The observation index folds these into a JSON String — see
+///     <see cref="MTConnectObservationIndex"/>. <see cref="IsStructured"/> stays <c>true</c> beside them: it is the
+///     wire-shape fact, this is the content.
+/// </param>
+/// <param name="ResetTriggered">
+///     <c>true</c> when the observation carried a non-empty <c>resetTriggered</c> attribute — the Agent's statement that
+///     the set was reset, so this observation <b>replaces</b> the accumulated set rather than merging into it.
+/// </param>
+/// <param name="IsDataSet">
+///     <c>true</c> when the element is data-set shaped: it carries the <c>count</c> attribute every DATA_SET / TABLE
+///     observation has (and no SAMPLE or ordinary EVENT does) or has <c>&lt;Entry&gt;</c> children. This — not
+///     <see cref="ResetTriggered"/> — is what lets the index serve an entry-less set as <c>{}</c> rather than as
+///     an absent value; <c>resetTriggered</c> also appears on ordinary accumulating SAMPLEs and must not matter there.
+/// </param>
 /// <param name="Condition">
 ///     The CONDITION-only detail (#626) — set exactly when the observation came from a
 ///     <c>&lt;Condition&gt;</c> container, <c>null</c> for every Sample/Event. Its presence is what marks an
@@ -179,7 +196,26 @@ public sealed record MTConnectObservation(
     bool IsStructured = false,
     int? SampleCount = null,
     double? SampleRate = null,
-    MTConnectConditionDetail? Condition = null);
+    MTConnectConditionDetail? Condition = null,
+    IReadOnlyList<MTConnectDataSetEntry>? Entries = null,
+    bool ResetTriggered = false,
+    bool IsDataSet = false);
+
+/// <summary>One <c>&lt;Cell key="…"&gt;</c> of a <c>TABLE</c> row (#650).</summary>
+public sealed record MTConnectDataSetCell(string Key, string Value);
+
+/// <summary>
+///     One <c>&lt;Entry key="…"&gt;</c> of a <c>DATA_SET</c> / <c>TABLE</c> observation (#650).
+/// </summary>
+/// <param name="Key">The entry's <c>key</c> attribute, verbatim.</param>
+/// <param name="Value">A <c>DATA_SET</c> entry's trimmed text; <c>null</c> for a <c>TABLE</c> row or a removed entry.</param>
+/// <param name="Removed"><c>true</c> when the entry carried <c>removed="true"</c>: the key is deleted from the set.</param>
+/// <param name="Cells">A <c>TABLE</c> row's cells in document order; <c>null</c> for a <c>DATA_SET</c> entry.</param>
+public sealed record MTConnectDataSetEntry(
+    string Key,
+    string? Value,
+    bool Removed,
+    IReadOnlyList<MTConnectDataSetCell>? Cells = null);
 
 /// <summary>
 ///     What an MTConnect CONDITION observation says beyond its state word (#626). Every field is the

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectObservationIndex.cs

@@ -1,6 +1,9 @@
 using System.Collections.Concurrent;
 using System.Collections.Frozen;
 using System.Globalization;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using System.Text.RegularExpressions;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 
 namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
@@ -50,7 +53,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
 ///     <para>
 ///         <b>Shapes this build does not represent</b> are coded <c>BadNotSupported</c> with a null
 ///         value rather than approximated: a non-Float64 array tag and a <c>DATA_SET</c> / <c>TABLE</c>
-///         observation (a Float64 <c>TIME_SERIES</c> vector is served as <c>double[]</c>, #627). The latter carries its
+///         observation whose children are not <c>&lt;Entry&gt;</c> elements (a Float64 <c>TIME_SERIES</c> vector is served as <c>double[]</c>, #627). The latter carries its
 ///         real content in <c>&lt;Entry key=…&gt;</c> child elements, which reach a text-valued
 ///         observation concatenated with the keys discarded — a two-entry set reads as the single
 ///         token <c>"12"</c> — and because the type inference correctly demotes those
@@ -111,6 +114,19 @@ internal sealed class MTConnectObservationIndex
         }.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase);
 
     private readonly ConcurrentDictionary<string, DataValueSnapshot> _snapshots = new(StringComparer.Ordinal);
+
+    // #650: the accumulated DATA_SET / TABLE content per dataItemId. A /sample chunk carries only the entries
+    // that changed, so the served JSON is the merge of the baseline and every delta since. Row value is the
+    // entry's text (DATA_SET) or its cell map (TABLE). Only the single pump (or a one-shot snapshot) calls
+    // Apply, but the lock keeps a concurrent one-shot read-through from corrupting a shared index.
+    private readonly Dictionary<string, SortedDictionary<string, SetEntry>> _sets = new(StringComparer.Ordinal);
+    private readonly object _setsLock = new();
+    private bool _anySets;
+
+    private sealed record SetEntry(string? Value, SortedDictionary<string, string>? Cells);
+
+    private static readonly Regex JsonNumber = new(
+        @"^-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
     private readonly FrozenDictionary<string, MTConnectTagDefinition> _tags;
     private readonly TimeProvider _timeProvider;
 
@@ -159,7 +175,11 @@ internal sealed class MTConnectObservationIndex
     /// <param name="result">The parsed streams document. An observation-free result is a legal
     /// keep-alive and leaves the index untouched.</param>
     /// <exception cref="ArgumentNullException"><paramref name="result"/> is <c>null</c>.</exception>
-    public void Apply(MTConnectStreamsResult result)
+    /// <param name="baseline">
+    ///     <c>true</c> for a <c>/current</c> document, which states the <b>whole</b> DATA_SET / TABLE and so replaces
+    ///     the accumulated set; <c>false</c> (default) for a <c>/sample</c> chunk, whose entries are deltas merged into it.
+    /// </param>
+    public void Apply(MTConnectStreamsResult result, bool baseline = false)
     {
         ArgumentNullException.ThrowIfNull(result);
 
@@ -170,9 +190,12 @@ internal sealed class MTConnectObservationIndex
         // — writing every observation and merging snapshots in place — would need a read-modify-write
         // per key and could expose a half-merged state to a concurrent reader.
         var resolved = new Dictionary<string, MTConnectObservation>(StringComparer.Ordinal);
-        foreach (var observation in result.Observations)
+        var seenSets = new HashSet<string>(StringComparer.Ordinal);
+        foreach (var raw in result.Observations)
         {
-            if (observation is null || string.IsNullOrWhiteSpace(observation.DataItemId)) continue;
+            if (raw is null || string.IsNullOrWhiteSpace(raw.DataItemId)) continue;
+
+            var observation = FoldDataSet(raw, baseline, seenSets);
 
             resolved[observation.DataItemId] =
                 resolved.TryGetValue(observation.DataItemId, out var incumbent)
@@ -217,7 +240,150 @@ internal sealed class MTConnectObservationIndex
     ///     <c>instanceId</c> changes, every value the index holds describes a device model that no
     ///     longer exists, and serving it would be worse than serving nothing.
     /// </summary>
-    public void Clear() => _snapshots.Clear();
+    public void Clear()
+    {
+        _snapshots.Clear();
+        lock (_setsLock) _sets.Clear();
+    }
+
+    /// <summary>
+    ///     DATA_SET / TABLE (#650). Folds an observation's <c>&lt;Entry&gt;</c> children into the accumulated set for
+    ///     its dataItemId and returns a plain String observation carrying the merged JSON; any other observation is
+    ///     returned untouched, except that an <c>UNAVAILABLE</c> one drops the accumulated set (a later delta must not
+    ///     resurrect pre-outage entries).
+    ///     <para>
+    ///         Semantics (MTConnect 2.x): a <c>/current</c> document (<paramref name="baseline"/>) or an observation
+    ///         with <c>resetTriggered</c> <b>replaces</b> the set; otherwise entries merge by key. <c>removed="true"</c>
+    ///         deletes the key (the key is absent from the JSON, never <c>null</c>). A TABLE row is replaced as a unit
+    ///         by a later entry for the same row key; a removed row key disappears. Several observations of one item in
+    ///         a single document fold in document order — only the first replaces on a baseline.
+    ///     </para>
+    /// </summary>
+    private MTConnectObservation FoldDataSet(MTConnectObservation observation, bool baseline, HashSet<string> seenSets)
+    {
+        var id = observation.DataItemId;
+
+        if (observation.Entries is not { Count: > 0 } entries)
+        {
+            var unavailable = string.IsNullOrWhiteSpace(observation.Value) && !observation.IsDataSet ||
+                              string.Equals(observation.Value, UnavailableSentinel, StringComparison.Ordinal);
+
+            if (unavailable)
+            {
+                // Hot path for every ordinary UNAVAILABLE observation: skip the lock until a set exists at all.
+                if (Volatile.Read(ref _anySets)) lock (_setsLock) _sets.Remove(id);
+
+                return observation;
+            }
+
+            if (observation.IsDataSet && !observation.IsStructured)
+            {
+                // An entry-less data-set-shaped observation (count="0", or resetTriggered) is the EMPTY set - a
+                // legitimate value, Good "{}", not an absence. A reset or a /current baseline discards what
+                // was accumulated; a bare empty delta leaves the set as it is.
+                string emptyJson;
+                lock (_setsLock)
+                {
+                    var first = seenSets.Add(id);
+                    if (!_sets.TryGetValue(id, out var set) || observation.ResetTriggered || (baseline && first))
+                    {
+                        set = new SortedDictionary<string, SetEntry>(StringComparer.Ordinal);
+                        _sets[id] = set;
+                        Volatile.Write(ref _anySets, true);
+                    }
+
+                    emptyJson = RenderSet(set);
+                }
+
+                return observation with { Value = emptyJson, ResetTriggered = false, IsDataSet = false };
+            }
+
+            return observation;
+        }
+
+        string json;
+        lock (_setsLock)
+        {
+            var first = seenSets.Add(id);
+            if (!_sets.TryGetValue(id, out var set) || observation.ResetTriggered || (baseline && first))
+            {
+                set = new SortedDictionary<string, SetEntry>(StringComparer.Ordinal);
+                _sets[id] = set;
+                Volatile.Write(ref _anySets, true);
+            }
+
+            foreach (var entry in entries)
+            {
+                if (entry.Removed)
+                {
+                    set.Remove(entry.Key);
+                }
+                else if (entry.Cells is not null)
+                {
+                    var cells = new SortedDictionary<string, string>(StringComparer.Ordinal);
+                    foreach (var cell in entry.Cells) cells[cell.Key] = cell.Value;
+                    set[entry.Key] = new SetEntry(null, cells);
+                }
+                else
+                {
+                    set[entry.Key] = new SetEntry(entry.Value ?? string.Empty, null);
+                }
+            }
+
+            json = RenderSet(set);
+        }
+
+        return observation with { Value = json, IsStructured = false, Entries = null, ResetTriggered = false, IsDataSet = false };
+    }
+
+    /// <summary>
+    ///     Compact, deterministic JSON: keys ordinal-sorted, no whitespace. Values that are strict JSON numbers are
+    ///     emitted as numbers with their original token (lossless — <c>007</c> and <c>+5</c> stay strings); everything
+    ///     else is a string.
+    /// </summary>
+    private static string RenderSet(SortedDictionary<string, SetEntry> set)
+    {
+        using var stream = new MemoryStream();
+        using (var w = new Utf8JsonWriter(stream, new JsonWriterOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }))
+        {
+            w.WriteStartObject();
+            foreach (var (key, entry) in set)
+            {
+                w.WritePropertyName(key);
+                if (entry.Cells is null)
+                {
+                    WriteScalar(w, entry.Value ?? string.Empty);
+                    continue;
+                }
+
+                w.WriteStartObject();
+                foreach (var (cellKey, cellValue) in entry.Cells)
+                {
+                    w.WritePropertyName(cellKey);
+                    WriteScalar(w, cellValue);
+                }
+
+                w.WriteEndObject();
+            }
+
+            w.WriteEndObject();
+        }
+
+        return System.Text.Encoding.UTF8.GetString(stream.ToArray());
+    }
+
+    private static void WriteScalar(Utf8JsonWriter w, string text)
+    {
+        if (JsonNumber.IsMatch(text) &&
+            double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var d) && double.IsFinite(d))
+        {
+            w.WriteRawValue(text);
+        }
+        else
+        {
+            w.WriteStringValue(text);
+        }
+    }
 
     private DateTime Now => _timeProvider.GetUtcNow().UtcDateTime;
 

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectStreamsParser.cs

@@ -363,11 +363,13 @@ internal static class MTConnectStreamsParser
     ///     <c>null</c> when there is none. Text found anywhere below an observation belongs to it —
     ///     that is what <c>XElement.Value</c> meant.
     /// </param>
+    /// <param name="entryCollector">The enclosing observation's DATA_SET / TABLE entry list (#650), or <c>null</c> outside one.</param>
     private static void ReadElement(
         XmlReader reader,
         ContainerKind parent,
         List<MTConnectObservation> observations,
-        ValueSink? textSink)
+        ValueSink? textSink,
+        List<MTConnectDataSetEntry>? entryCollector = null)
     {
         var isObservation = parent != ContainerKind.None;
         var isCondition = parent == ContainerKind.Condition;
@@ -379,6 +381,9 @@ internal static class MTConnectStreamsParser
         string? conditionState = null;
         int? sampleCount = null;
         double? sampleRate = null;
+        var resetTriggered = false;
+        var hasCount = false;
+        List<MTConnectDataSetEntry>? entries = null;
         (string? NativeCode, string? NativeSeverity, string? Qualifier, string? Type) conditionAttributes = default;
 
         if (isObservation)
@@ -405,6 +410,9 @@ internal static class MTConnectStreamsParser
             // against the declared sampleCount only when one parses).
             if (!isCondition)
             {
+                resetTriggered = !string.IsNullOrWhiteSpace(MTConnectXml.OptionalAttribute(reader, "resetTriggered"));
+                hasCount = MTConnectXml.OptionalAttribute(reader, "count") is not null;
+                entries = [];
                 sampleCount = int.TryParse(
                     MTConnectXml.OptionalAttribute(reader, "sampleCount"),
                     NumberStyles.Integer,
@@ -431,7 +439,7 @@ internal static class MTConnectStreamsParser
         // enclosing observation's sink: a condition is never nested inside a text-valued observation.
         var sink = isObservation ? new ValueSink() : textSink;
 
-        var hasElements = !reader.IsEmptyElement && ReadChildren(reader, kind, observations, sink);
+        var hasElements = !reader.IsEmptyElement && ReadChildren(reader, kind, observations, sink, entries ?? entryCollector);
 
         if (!isObservation)
         {
@@ -475,7 +483,13 @@ internal static class MTConnectStreamsParser
             IsStructured: !isCondition && hasElements,
             SampleCount: sampleCount,
             SampleRate: sampleRate,
-            Condition: condition);
+            Condition: condition,
+            Entries: entries is { Count: > 0 } ? entries : null,
+
+            // resetTriggered only means "replace the set" on a data-set-shaped observation; on an ordinary
+            // accumulating SAMPLE/EVENT (PartCount resetTriggered="DAILY") it is not our concern.
+            ResetTriggered: resetTriggered && (hasCount || entries is { Count: > 0 }),
+            IsDataSet: !isCondition && (hasCount || entries is { Count: > 0 }));
     }
 
     /// <summary>
@@ -486,7 +500,8 @@ internal static class MTConnectStreamsParser
         XmlReader reader,
         ContainerKind kind,
         List<MTConnectObservation> observations,
-        ValueSink? sink)
+        ValueSink? sink,
+        List<MTConnectDataSetEntry>? entries)
     {
         var hasElements = false;
 
@@ -496,7 +511,17 @@ internal static class MTConnectStreamsParser
             {
                 case XmlNodeType.Element:
                     hasElements = true;
-                    ReadElement(reader, kind, observations, sink);
+
+                    // #650: an <Entry> directly under a non-condition observation is DATA_SET / TABLE
+                    // content, captured structurally rather than as concatenated text.
+                    if (entries is not null && kind == ContainerKind.None && MTConnectXml.IsNamed(reader, "Entry"))
+                    {
+                        ReadEntry(reader, entries, sink);
+                    }
+                    else
+                    {
+                        ReadElement(reader, kind, observations, sink, entries);
+                    }
 
                     break;
 
@@ -520,6 +545,60 @@ internal static class MTConnectStreamsParser
         return hasElements;
     }
 
+    /// <summary>
+    ///     Consumes one <c>&lt;Entry&gt;</c> (and its <c>&lt;Cell&gt;</c> children, for a TABLE row), honouring the
+    ///     <see cref="ReadChildren"/> cursor contract: returns on the entry's end tag, or on the entry itself when empty.
+    ///     An entry with no <c>key</c> is dropped — it cannot be addressed.
+    /// </summary>
+    private static void ReadEntry(XmlReader reader, List<MTConnectDataSetEntry> entries, ValueSink? sink)
+    {
+        var key = MTConnectXml.OptionalAttribute(reader, "key");
+        var removed = string.Equals(MTConnectXml.OptionalAttribute(reader, "removed")?.Trim(), "true", StringComparison.OrdinalIgnoreCase);
+        var empty = reader.IsEmptyElement;
+        var text = new System.Text.StringBuilder();
+        List<MTConnectDataSetCell>? cells = null;
+
+        while (!empty && reader.Read())
+        {
+            if (reader.NodeType == XmlNodeType.EndElement) break;
+
+            if (reader.NodeType == XmlNodeType.Element && MTConnectXml.IsNamed(reader, "Cell"))
+            {
+                cells ??= [];
+                var cellKey = MTConnectXml.OptionalAttribute(reader, "key");
+                var cellText = new System.Text.StringBuilder();
+                if (!reader.IsEmptyElement)
+                {
+                    while (reader.Read() && reader.NodeType != XmlNodeType.EndElement)
+                    {
+                        if (reader.NodeType is XmlNodeType.Text or XmlNodeType.CDATA or XmlNodeType.SignificantWhitespace)
+                            { cellText.Append(reader.Value); sink?.Append(reader.Value); }
+                        else if (reader.NodeType == XmlNodeType.Element)
+                            reader.Skip();
+                    }
+                }
+
+                if (!string.IsNullOrEmpty(cellKey)) cells.Add(new MTConnectDataSetCell(cellKey, cellText.ToString().Trim()));
+            }
+            else if (reader.NodeType is XmlNodeType.Text or XmlNodeType.CDATA or XmlNodeType.SignificantWhitespace)
+            {
+                { text.Append(reader.Value); sink?.Append(reader.Value); }
+            }
+            else if (reader.NodeType == XmlNodeType.Element)
+            {
+                reader.Skip();
+            }
+        }
+
+        if (string.IsNullOrEmpty(key)) return;
+
+        entries.Add(removed
+            ? new MTConnectDataSetEntry(key, null, true)
+            : cells is not null
+                ? new MTConnectDataSetEntry(key, null, false, cells)
+                : new MTConnectDataSetEntry(key, text.ToString().Trim(), false));
+    }
+
     /// <summary>
     ///     A condition's state is its element name. <c>Unavailable</c> is normalized to the
     ///     <see cref="UnavailableSentinel"/> spelling so downstream quality mapping has exactly one