lmxopcua/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectStreamsParser.cs

using System.Globalization;
using System.Runtime.InteropServices;
using System.Text;
using System.Xml;
using System.Xml.Linq;

namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;

/// <summary>
///     Turns an MTConnect Agent <c>MTConnectStreams</c> document — a <c>/current</c> snapshot or one
///     chunk of a <c>/sample</c> multipart stream — into the neutral
///     <see cref="MTConnectStreamsResult"/>. Deliberately a pure, socket-free static so the whole
///     parse is unit-testable against canned fixtures; chunk framing lives in
///     <see cref="MTConnectAgentClient"/> and hands this parser one chunk's XML at a time.
/// </summary>
/// <remarks>
///     <para>
///         <b>Why hand-rolled rather than TrakHound.</b> Same finding as
///         <see cref="MTConnectProbeParser"/>: the packages pinned by Task 0 ship no XML document
///         formatter, and TrakHound's streaming client offers no socket-free entry point. Both
///         package references were dropped in Task 7; the driver depends only on the BCL.
///     </para>
///     <para>
///         <b>The Streams document is shaped nothing like the Devices document</b>, and three
///         differences drive this parser's design.
///     </para>
///     <list type="number">
///         <item>
///             <description>
///                 <b>The observation element's name is the data item's type in PascalCase</b>
///                 (<c>Position</c>, <c>PartCount</c>, <c>Execution</c>) — not the probe's
///                 UPPER_SNAKE <c>POSITION</c> / <c>PART_COUNT</c>. The standard defines hundreds of
///                 types and vendors add more, so matching a fixed set of element names would drop
///                 observations silently. <b>Every</b> element child of <c>&lt;Samples&gt;</c>,
///                 <c>&lt;Events&gt;</c> and <c>&lt;Condition&gt;</c> is therefore an observation,
///                 keyed by its <c>dataItemId</c> attribute — the only identity the probe model and
///                 the streams document actually share.
///             </description>
///         </item>
///         <item>
///             <description>
///                 <b>A CONDITION observation's value is its ELEMENT NAME, not its text.</b>
///                 <c>&lt;Normal/&gt;</c> / <c>&lt;Warning/&gt;</c> / <c>&lt;Fault/&gt;</c> /
///                 <c>&lt;Unavailable/&gt;</c> carry the state in the name and are typically empty;
///                 where text is present it is the operator-facing message, not the state. Reading
///                 the text would report every condition as an empty string.
///                 <c>&lt;Unavailable/&gt;</c> is normalized to the literal
///                 <see cref="UnavailableSentinel"/> so a no-comms condition lands on exactly the
///                 same sentinel as a Sample/Event whose text is <c>UNAVAILABLE</c> — the
///                 observation index maps that one token to bad quality.
///             </description>
///         </item>
///         <item>
///             <description>
///                 <b>Timestamps are normalized to UTC explicitly.</b> MTConnect timestamps are
///                 ISO-8601 Zulu, but a bare <c>DateTime.Parse</c> yields <c>Local</c> or
///                 <c>Unspecified</c> depending on machine settings — and this value becomes the
///                 OPC UA SourceTimestamp, so a wrong <see cref="DateTimeKind"/> shifts every
///                 timestamp by the host's UTC offset with nothing to show for it.
///             </description>
///         </item>
///     </list>
///     <para>
///         <b>The read is forward-only.</b> A <c>/sample</c> chunk is parsed once and thrown away,
///         so it is read straight off an <see cref="XmlReader"/> rather than through an
///         <see cref="XDocument"/>: on the pump's hot path the tree was pure garbage — one
///         <c>XElement</c> plus its attributes and text nodes per observation, per chunk, per
///         second — and nothing here needs a second look at a node it has already passed. The
///         extraction is unchanged; the two rules that keep it honest are that whitespace-only text
///         nodes are dropped exactly as <see cref="XDocument"/> dropped them (see
///         <see cref="ReaderSettings"/>), and that an observation's slot in the result list is
///         reserved <i>before</i> its subtree is read so observations still arrive in document
///         order. The one place a tree is still built is the cold non-<c>MTConnectStreams</c> root
///         below, where the Agent's own error text has to be lifted out of a document this parser
///         is otherwise finished with.
///     </para>
///     <para>
///         <b>One thing did change, and only for a document that is wrong twice over.</b> A missing
///         <c>&lt;Header&gt;</c> used to be reported ahead of a malformed observation, because the
///         tree could be interrogated in whatever order suited; forward-only, the header is not
///         known to be missing until the root's children have run out, so the observation is
///         reported first. Both are <see cref="InvalidDataException"/>, and nothing that is wrong in
///         only one way reports differently than it did.
///     </para>
///     <para>
///         <b>Failure posture.</b> A malformed document, a non-<c>MTConnectStreams</c> root (an Agent
///         <c>MTConnectError</c> served under HTTP 200 included), an unusable header, or an
///         observation missing its <c>dataItemId</c>/<c>timestamp</c> all throw
///         <see cref="InvalidDataException"/>. A document with a valid header and <i>zero</i>
///         observations, by contrast, is entirely legal — <c>/sample</c> chunks are deltas and an
///         idle Agent's keep-alive is an observation-free document — so it parses to an empty
///         observation list rather than throwing.
///     </para>
/// </remarks>
internal static class MTConnectStreamsParser
{
    /// <summary>
    ///     The single token that means "the Agent has no value for this data item". Sample/Event
    ///     observations carry it as text; a CONDITION carries it as the element name
    ///     <c>&lt;Unavailable/&gt;</c> and is normalized onto this spelling here.
    /// </summary>
    private const string UnavailableSentinel = "UNAVAILABLE";

    private const string ConditionContainer = "Condition";

    /// <summary>How <see cref="MTConnectXml"/>'s shared error messages name this document.</summary>
    private const string Subject = "streams response";

    /// <summary>How those messages name the header element.</summary>
    private const string HeaderOwner = "the <Header>";

    /// <summary>
    ///     How the whole document is read. Two settings are load-bearing rather than incidental.
    /// </summary>
    /// <remarks>
    ///     <para>
    ///         <b><see cref="XmlReaderSettings.IgnoreWhitespace"/> reproduces
    ///         <see cref="XDocument"/>'s default.</b> <c>XDocument.Load</c>/<c>Parse</c> without
    ///         <see cref="LoadOptions.PreserveWhitespace"/> sets exactly this, so a whitespace-only
    ///         text node never became an <c>XText</c> and never reached <c>XElement.Value</c>.
    ///         Leaving it at the reader's own default would silently start folding indentation into
    ///         observation values that contain element children — <c>"1 2"</c> where the tree read
    ///         <c>"12"</c>.
    ///     </para>
    ///     <para>
    ///         <b>A DTD is prohibited</b> because a streams document is untrusted network input and
    ///         entity expansion is not a feature this parser has any use for. This too matches what
    ///         <see cref="XDocument"/> defaulted to.
    ///     </para>
    /// </remarks>
    private static readonly XmlReaderSettings ReaderSettings = new()
    {
        IgnoreWhitespace = true,
        DtdProcessing = DtdProcessing.Prohibit,

        // The caller owns the stream it handed in; disposing the reader must not close it.
        CloseInput = false
    };

    /// <summary>Parses a <c>/current</c> response, or one <c>/sample</c> chunk, held as a string.</summary>
    /// <param name="xml">The raw <c>MTConnectStreams</c> XML document.</param>
    /// <exception cref="InvalidDataException">
    ///     The payload is empty, not well-formed XML, not an <c>MTConnectStreams</c> document (an
    ///     Agent <c>MTConnectError</c> included — Agents answer a bad request with one under HTTP
    ///     200), carries no usable <c>&lt;Header&gt;</c>, or carries a malformed observation.
    /// </exception>
    public static MTConnectStreamsResult Parse(string xml)
    {
        if (string.IsNullOrWhiteSpace(xml))
        {
            throw new InvalidDataException(
                "MTConnect streams response was empty; expected an MTConnectStreams XML document.");
        }

        // A chunk lifted out of a multipart frame can carry a byte-order mark or the framing's
        // trailing line break; XmlReader rejects either ahead of the XML declaration.
        using var text = new StringReader(xml.Trim('\uFEFF', ' ', '\t', '\r', '\n'));
        using var reader = XmlReader.Create(text, ReaderSettings);

        return Read(reader);
    }

    /// <summary>
    ///     Parses one <c>/sample</c> chunk straight from the bytes the multipart framer produced.
    /// </summary>
    /// <remarks>
    ///     The <c>/sample</c> hot path used to materialise every chunk twice — once as a decoded
    ///     string, once again as the trimmed copy handed to the parser — for a document that is read
    ///     once and thrown away. Parsing from the bytes drops both copies; <see cref="XmlReader"/>
    ///     does its own UTF-8 decoding off the stream. The framing tolerance the string overload's
    ///     <c>Trim</c> provided is preserved by <see cref="TrimFraming"/>, which slices rather than
    ///     copies.
    /// </remarks>
    /// <param name="xml">The chunk's raw bytes; a UTF-8 BOM and surrounding framing whitespace are tolerated.</param>
    /// <exception cref="InvalidDataException">As for <see cref="Parse(string)"/>.</exception>
    public static MTConnectStreamsResult Parse(ReadOnlyMemory<byte> xml)
    {
        var trimmed = TrimFraming(xml);
        if (trimmed.IsEmpty)
        {
            throw new InvalidDataException(
                "MTConnect streams response was empty; expected an MTConnectStreams XML document.");
        }

        using var stream = AsStream(trimmed);

        return Parse(stream);
    }

    /// <summary>Parses a <c>/current</c> response read from a stream.</summary>
    /// <param name="stream">A stream positioned at the start of the XML document.</param>
    /// <exception cref="InvalidDataException">As for <see cref="Parse(string)"/>.</exception>
    public static MTConnectStreamsResult Parse(Stream stream)
    {
        ArgumentNullException.ThrowIfNull(stream);

        using var reader = XmlReader.Create(stream, ReaderSettings);

        return Read(reader);
    }

    /// <summary>
    ///     Slices off what the string overload's <c>Trim('', ' ', '\t', '\r', '\n')</c> removed:
    ///     a UTF-8 byte-order mark and the multipart framing's surrounding line breaks, either of
    ///     which <see cref="XmlReader"/> rejects ahead of the XML declaration.
    /// </summary>
    private static ReadOnlyMemory<byte> TrimFraming(ReadOnlyMemory<byte> xml)
    {
        var span = xml.Span;
        var start = 0;
        var end = span.Length;

        while (start < end)
        {
            if (end - start >= 3 && span[start] == 0xEF && span[start + 1] == 0xBB && span[start + 2] == 0xBF)
            {
                start += 3;

                continue;
            }

            if (!IsFramingWhitespace(span[start]))
            {
                break;
            }

            start++;
        }

        while (end > start && IsFramingWhitespace(span[end - 1]))
        {
            end--;
        }

        return xml[start..end];
    }

    private static bool IsFramingWhitespace(byte value) =>
        value is (byte)' ' or (byte)'\t' or (byte)'\r' or (byte)'\n';

    /// <summary>
    ///     Views the (already trimmed) bytes as a stream without copying them where the memory is
    ///     array-backed — which it always is on the <c>/sample</c> path, whose framer hands over the
    ///     part's own <c>byte[]</c>. That branch <b>aliases</b> the caller's array; only the
    ///     non-array-backed fallback copies (via <c>ToArray</c>).
    ///     <para>
    ///         Ownership, so a caller can reason about the aliased buffer: nothing on this path pools
    ///         or rents it (the framer allocates each <c>byte[]</c> fresh), and parsing is fully
    ///         synchronous, so the buffer need only stay valid and unmutated for the duration of the
    ///         <c>Parse</c> call. The returned <see cref="MTConnectStreamsResult"/> holds no reference
    ///         into it — <see cref="XmlReader"/> materialises its own strings — so the caller may
    ///         reuse or discard the buffer as soon as <c>Parse</c> returns.
    ///     </para>
    /// </summary>
    private static MemoryStream AsStream(ReadOnlyMemory<byte> xml) =>
        MemoryMarshal.TryGetArray(xml, out ArraySegment<byte> segment) && segment.Array is not null
            ? new MemoryStream(segment.Array, segment.Offset, segment.Count, writable: false)
            : new MemoryStream(xml.ToArray(), writable: false);

    /// <summary>
    ///     Reads the whole document, translating the reader's own well-formedness complaints into
    ///     this parser's failure family. A forward-only read raises them wherever it happens to
    ///     reach the damage rather than all at once up front, so the wrapper spans the entire walk.
    /// </summary>
    private static MTConnectStreamsResult Read(XmlReader reader)
    {
        try
        {
            return ReadDocument(reader);
        }
        catch (XmlException ex)
        {
            throw new InvalidDataException($"MTConnect streams response is not well-formed XML: {ex.Message}", ex);
        }
    }

    private static MTConnectStreamsResult ReadDocument(XmlReader reader)
    {
        while (reader.Read() && reader.NodeType != XmlNodeType.Element)
        {
            // Skip the XML declaration, comments and processing instructions ahead of the root.
        }

        if (reader.NodeType != XmlNodeType.Element)
        {
            throw new InvalidDataException("MTConnect streams response has no root element.");
        }

        if (!MTConnectXml.IsNamed(reader, "MTConnectStreams"))
        {
            // The cold path, and the one place a tree is still built: an MTConnectError arrives in
            // exactly this shape (Agents serve one under HTTP 200), and its <Error> elements — the
            // Agent's own account of what it objected to — are the most useful thing in the whole
            // response. Materialising the root is how the shared extraction can reach them.
            throw new InvalidDataException(
                MTConnectXml.DescribeUnexpectedRoot((XElement)XNode.ReadFrom(reader), "MTConnectStreams", Subject));
        }

        var observations = new List<MTConnectObservation>();
        HeaderSequences? header = null;

        if (!reader.IsEmptyElement)
        {
            while (reader.Read() && reader.NodeType != XmlNodeType.EndElement)
            {
                if (reader.NodeType != XmlNodeType.Element)
                {
                    continue;
                }

                // The <Header> is a direct child of the root, and the first one wins — but it is
                // still walked like any other element rather than skipped, so a container nested
                // inside it yields its observations exactly as the whole-subtree descent did.
                if (header is null && MTConnectXml.IsNamed(reader, "Header"))
                {
                    header = ReadHeader(reader);
                }

                ReadElement(reader, ContainerKind.None, observations, textSink: null);
            }
        }

        var sequences = header ?? throw new InvalidDataException(
            "MTConnect streams response has no <Header>; without it the sequence bookkeeping the sample pump runs on is unknowable.");

        return new MTConnectStreamsResult(
            sequences.InstanceId,
            sequences.NextSequence,
            sequences.FirstSequence,
            observations);
    }

    /// <summary>
    ///     The three sequence numbers the sample pump runs on, read off the <c>&lt;Header&gt;</c>'s
    ///     start tag. Read eagerly, where the header is met, so an unusable header is still reported
    ///     ahead of anything the <c>&lt;Streams&gt;</c> that follows it might also be wrong about.
    /// </summary>
    private static HeaderSequences ReadHeader(XmlReader reader) =>
        new(
            MTConnectXml.RequiredLongAttribute(reader, "instanceId", HeaderOwner, Subject),
            MTConnectXml.RequiredLongAttribute(reader, "nextSequence", HeaderOwner, Subject),
            MTConnectXml.RequiredLongAttribute(reader, "firstSequence", HeaderOwner, Subject));

    /// <summary>
    ///     Consumes the element the reader is positioned on, collecting any observation it is and
    ///     any the rest of its subtree holds. Every element is descended into — observations are
    ///     found by their <i>parent</i> being one of the three containers, not by walking a fixed
    ///     DeviceStream → ComponentStream chain, so a vendor's extra nesting level cannot silently
    ///     hide a device's observations.
    /// </summary>
    /// <remarks>
    ///     <b>Cursor contract:</b> on return the reader sits on the element's
    ///     <see cref="XmlNodeType.EndElement"/>, or on the element itself when it was written empty
    ///     — either way the caller's next <see cref="XmlReader.Read"/> lands on whatever follows.
    ///     Every recursion below relies on that, which is why the children are walked by hand
    ///     rather than through <c>Skip</c>/<c>ReadSubtree</c>.
    /// </remarks>
    /// <param name="reader">A reader positioned on the element's start tag.</param>
    /// <param name="parent">What the enclosing element is, which decides whether this one is an observation.</param>
    /// <param name="observations">Collects the observations found, in document order.</param>
    /// <param name="textSink">
    ///     The text accumulator of the nearest enclosing observation whose value is its text, or
    ///     <c>null</c> when there is none. Text found anywhere below an observation belongs to it —
    ///     that is what <c>XElement.Value</c> meant.
    /// </param>
    private static void ReadElement(
        XmlReader reader,
        ContainerKind parent,
        List<MTConnectObservation> observations,
        ValueSink? textSink)
    {
        var isObservation = parent != ContainerKind.None;
        var isCondition = parent == ContainerKind.Condition;
        var kind = ContainerKindOf(reader);

        var slot = -1;
        string? dataItemId = null;
        var timestampUtc = default(DateTime);
        string? conditionState = null;

        if (isObservation)
        {
            var name = reader.LocalName;
            dataItemId = MTConnectXml.RequiredAttribute(reader, "dataItemId", $"Observation <{name}>", Subject);
            timestampUtc = ReadTimestampUtc(reader, dataItemId);
            conditionState = isCondition ? ConditionState(name) : null;

            // The slot is reserved BEFORE the subtree is read, and filled after: an observation is
            // only complete once its text and its child-element flag are known, but appending it
            // then would report a nested observation ahead of the one enclosing it. The tree walk
            // this replaced was document-order, and callers key snapshots off that order.
            slot = observations.Count;
            observations.Add(null!);
        }

        // A CONDITION's value is its element NAME, so nothing below it is collected for it — but an
        // enclosing observation's text still is, hence the ancestor sink rather than none at all.
        var sink = isObservation && !isCondition ? new ValueSink() : textSink;

        var hasElements = !reader.IsEmptyElement && ReadChildren(reader, kind, observations, sink);

        if (!isObservation)
        {
            return;
        }

        string value;
        if (isCondition)
        {
            value = conditionState!;
        }
        else
        {
            var raw = sink!.Value;

            // An observation nested inside another one contributes its text to the outer one too,
            // which is what concatenating every descendant text node amounted to.
            textSink?.Append(raw);
            value = raw.Trim();
        }

        observations[slot] = new MTConnectObservation(
            dataItemId!,
            value,
            timestampUtc,

            // A DATA_SET/TABLE observation keeps its content in <Entry key="…"> children, so the
            // text read above concatenates them into nonsense ("12" for two entries). Flagged here
            // because this is the only layer that can still see the distinction — see
            // MTConnectObservation.IsStructured. Never true for a CONDITION: its value comes from
            // the element NAME, so child content cannot corrupt it, and flagging one would throw
            // away a perfectly well-determined state.
            IsStructured: !isCondition && hasElements);
    }

    /// <summary>
    ///     Walks a non-empty element's children through its <see cref="XmlNodeType.EndElement"/>,
    ///     reporting whether any of them was an element.
    /// </summary>
    private static bool ReadChildren(
        XmlReader reader,
        ContainerKind kind,
        List<MTConnectObservation> observations,
        ValueSink? sink)
    {
        var hasElements = false;

        while (reader.Read())
        {
            switch (reader.NodeType)
            {
                case XmlNodeType.Element:
                    hasElements = true;
                    ReadElement(reader, kind, observations, sink);

                    break;

                // Deliberately not XmlNodeType.Whitespace: ReaderSettings suppresses it, and were it
                // ever surfaced it would still not belong in the value — the tree this replaced
                // dropped whitespace-only nodes on exactly the same setting.
                case XmlNodeType.Text:
                case XmlNodeType.CDATA:
                case XmlNodeType.SignificantWhitespace:
                    sink?.Append(reader.Value);

                    break;

                case XmlNodeType.EndElement:
                    return hasElements;
            }
        }

        // Unreachable on any input the reader accepts: running out of document inside an open
        // element is a well-formedness error, which the reader raises from Read() itself.
        return hasElements;
    }

    /// <summary>
    ///     A condition's state is its element name. <c>Unavailable</c> is normalized to the
    ///     <see cref="UnavailableSentinel"/> spelling so downstream quality mapping has exactly one
    ///     token to recognize; every other state is reported as-named (<c>Normal</c>, <c>Warning</c>,
    ///     <c>Fault</c>, and any vendor state).
    /// </summary>
    private static string ConditionState(string localName) =>
        string.Equals(localName, "Unavailable", StringComparison.OrdinalIgnoreCase) ? UnavailableSentinel : localName;

    private static DateTime ReadTimestampUtc(XmlReader reader, string dataItemId)
    {
        var raw = MTConnectXml.RequiredAttribute(reader, "timestamp", $"Observation '{dataItemId}'", Subject);

        if (!DateTime.TryParse(
                raw,
                CultureInfo.InvariantCulture,
                DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal,
                out var parsed))
        {
            throw new InvalidDataException(
                $"MTConnect streams response is malformed: observation '{dataItemId}' has an unparseable 'timestamp' ('{raw}').");
        }

        // AdjustToUniversal already yields Utc; stated explicitly so a future styles change cannot
        // quietly hand the OPC UA layer a Local or Unspecified SourceTimestamp.
        return DateTime.SpecifyKind(parsed, DateTimeKind.Utc);
    }

    /// <summary>
    ///     Which of the three observation containers — if any — the element the reader is positioned
    ///     on is. A ComponentStream may carry any subset of them, including none.
    /// </summary>
    private static ContainerKind ContainerKindOf(XmlReader reader)
    {
        if (MTConnectXml.IsNamed(reader, ConditionContainer))
        {
            return ContainerKind.Condition;
        }

        return MTConnectXml.IsNamed(reader, "Samples") || MTConnectXml.IsNamed(reader, "Events")
            ? ContainerKind.Values
            : ContainerKind.None;
    }

    /// <summary>What an element's children are: nothing special, valued observations, or conditions.</summary>
    private enum ContainerKind
    {
        /// <summary>Not an observation container; its element children are just more structure.</summary>
        None,

        /// <summary><c>&lt;Samples&gt;</c> or <c>&lt;Events&gt;</c> — children take their value from their text.</summary>
        Values,

        /// <summary><c>&lt;Condition&gt;</c> — children take their value from their element NAME.</summary>
        Condition
    }

    /// <summary>The three sequence numbers read off a <c>&lt;Header&gt;</c>.</summary>
    private readonly record struct HeaderSequences(long InstanceId, long NextSequence, long FirstSequence);

    /// <summary>
    ///     Accumulates one observation's text across however many nodes the parser hands it, without
    ///     a <see cref="StringBuilder"/> for the single-text-node shape every ordinary scalar
    ///     observation has — which is the shape this parser sees thousands of times a second.
    /// </summary>
    private sealed class ValueSink
    {
        private string? _single;
        private StringBuilder? _builder;

        /// <summary>Everything appended so far, untrimmed — <c>XElement.Value</c>'s concatenation.</summary>
        public string Value => _builder?.ToString() ?? _single ?? string.Empty;

        public void Append(string text)
        {
            if (_builder is not null)
            {
                _builder.Append(text);

                return;
            }

            if (_single is null)
            {
                _single = text;

                return;
            }

            _builder = new StringBuilder(_single).Append(text);
            _single = null;
        }
    }
}