feat(commons): T0-2 RFC-4180 CSV parser + writer
Add a pure, no-I/O RFC 4180 CSV reader and writer to the Commons project (greenfield — no CSV code existed). CsvParser: string / TextReader -> rows of string[]; streaming IEnumerable overload + eager Parse(string) + thin ParseWithHeader. Handles quoted fields (embedded delimiter/CR/LF/CRLF, "" -> " escape), space preservation, CRLF/LF/CR terminators, no phantom trailing row. Strict malformed-input policy: FormatException with 1-based line/column on a quote inside an unquoted field, a char after a closing quote, or an unterminated quote. Faithful empty-line policy: a blank line is one empty field (callers filter). CsvWriter: rows -> string / TextWriter; quote-on-demand (delimiter, quote, CR, LF only; internal quotes doubled), configurable delimiter/newline (default CRLF), optional quote-all, no trailing terminator. Tests (xUnit + Shouldly, 109): full RFC edge corpus for the parser, the writer quote-on-demand rules, and the required Parse(Write(rows)) == rows round-trip property over a table of every tricky character (x CRLF/LF/ quote-all/semicolon). Claude-Session: https://claude.ai/code/session_01LVneM3eh1UtJxEisFXgmox
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,293 @@
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.Csv;
|
||||
|
||||
/// <summary>
|
||||
/// A pure, allocation-lean RFC 4180 CSV reader with no file/stream I/O of its own — it consumes a
|
||||
/// <see cref="string"/> or a <see cref="TextReader"/> and yields rows of fields. This is the single
|
||||
/// CSV-reading authority in the tree; keep the RFC contract pinned here (and mirrored in
|
||||
/// <see cref="CsvWriter"/>) rather than re-deriving it at each call site.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para><b>Faithful to RFC 4180.</b> The grammar handled:</para>
|
||||
/// <list type="bullet">
|
||||
/// <item>Fields are separated by the delimiter (default <c>','</c>); records are separated by
|
||||
/// newlines.</item>
|
||||
/// <item>A field may be quoted with double-quotes (<c>"</c>). A quoted field may contain the
|
||||
/// delimiter, CR, LF, and CRLF literally, and represents a literal double-quote by doubling it
|
||||
/// (<c>""</c> → <c>"</c>).</item>
|
||||
/// <item>An unquoted field runs literally up to the next delimiter or newline; leading and
|
||||
/// trailing spaces are preserved (RFC 4180 §2.4 — spaces are part of a field).</item>
|
||||
/// <item>CRLF, bare LF, and bare CR are all accepted as record terminators.</item>
|
||||
/// </list>
|
||||
/// <para><b>Empty-line policy.</b> Faithful to the RFC: a line with no characters yields a single row
|
||||
/// containing one empty field (<c>[""]</c>); it is NOT silently dropped. Callers that want blank lines
|
||||
/// skipped should filter the result (e.g. <c>rows.Where(r => r.Length > 1 || r[0].Length > 0)</c>).
|
||||
/// A trailing record terminator at end-of-input does NOT produce a phantom empty final row, and its
|
||||
/// absence does not lose the last row.</para>
|
||||
/// <para><b>Malformed-input policy.</b> This parser is <b>strict</b>. It throws
|
||||
/// <see cref="System.FormatException"/> (carrying a 1-based line/column position) for the malformed
|
||||
/// shapes RFC 4180 forbids: (1) a bare double-quote inside an otherwise-unquoted field (e.g.
|
||||
/// <c>ab"c</c>), (2) a stray character after a closing quote other than the delimiter or a newline
|
||||
/// (e.g. <c>"ab"c</c>), and (3) an unterminated quoted field at end-of-input. Strictness is deliberate
|
||||
/// — silent lenient recovery hides data-shape bugs in imported files.</para>
|
||||
/// </remarks>
|
||||
public static class CsvParser
|
||||
{
|
||||
private const char Quote = '"';
|
||||
private const char Cr = '\r';
|
||||
private const char Lf = '\n';
|
||||
|
||||
/// <summary>
|
||||
/// Parses <paramref name="text"/> fully into rows of fields. Convenience wrapper over
|
||||
/// <see cref="Parse(TextReader,char)"/>; materialises the whole document.
|
||||
/// </summary>
|
||||
/// <param name="text">The CSV document. <c>null</c> is treated as empty.</param>
|
||||
/// <param name="delimiter">The field separator (default comma).</param>
|
||||
/// <returns>The rows, each an array of field values. Empty input yields zero rows.</returns>
|
||||
/// <exception cref="System.FormatException">The input violates the strict RFC 4180 grammar.</exception>
|
||||
public static IReadOnlyList<string[]> Parse(string? text, char delimiter = ',')
|
||||
{
|
||||
using var reader = new StringReader(text ?? string.Empty);
|
||||
var rows = new List<string[]>();
|
||||
foreach (var row in Parse(reader, delimiter))
|
||||
{
|
||||
rows.Add(row);
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Streams rows from <paramref name="reader"/> lazily — a single forward pass, one row
|
||||
/// materialised at a time. The reader is not disposed by this method.
|
||||
/// </summary>
|
||||
/// <param name="reader">The source. Read to end-of-input.</param>
|
||||
/// <param name="delimiter">The field separator (default comma).</param>
|
||||
/// <returns>A lazily-evaluated sequence of rows; each row is a freshly allocated field array.</returns>
|
||||
/// <exception cref="System.ArgumentNullException"><paramref name="reader"/> is <c>null</c>.</exception>
|
||||
/// <exception cref="System.ArgumentException"><paramref name="delimiter"/> is a quote, CR, or LF.</exception>
|
||||
/// <exception cref="System.FormatException">The input violates the strict RFC 4180 grammar.</exception>
|
||||
public static IEnumerable<string[]> Parse(TextReader reader, char delimiter = ',')
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(reader);
|
||||
if (delimiter is Quote or Cr or Lf)
|
||||
{
|
||||
throw new ArgumentException("Delimiter must not be a double-quote, CR, or LF.", nameof(delimiter));
|
||||
}
|
||||
|
||||
return Iterate(reader, delimiter);
|
||||
}
|
||||
|
||||
private static IEnumerable<string[]> Iterate(TextReader reader, char delimiter)
|
||||
{
|
||||
var field = new StringBuilder();
|
||||
var row = new List<string>();
|
||||
|
||||
// True once the current row has produced any field boundary or content — i.e. once we've seen a
|
||||
// char that commits us to emitting at least one field. Reset to false right after a record
|
||||
// terminator closes a row, so a trailing newline at EOF does NOT synthesise a phantom row.
|
||||
var rowOpen = false;
|
||||
|
||||
// 1-based cursor, maintained for FormatException messages.
|
||||
var line = 1;
|
||||
var col = 0;
|
||||
|
||||
int read;
|
||||
while ((read = reader.Read()) != -1)
|
||||
{
|
||||
var c = (char)read;
|
||||
col++;
|
||||
|
||||
if (c == Quote)
|
||||
{
|
||||
if (field.Length != 0)
|
||||
{
|
||||
throw new FormatException(
|
||||
$"Unexpected double-quote inside an unquoted field at line {line}, column {col}. " +
|
||||
"A field is quoted only when the quote is its first character.");
|
||||
}
|
||||
|
||||
rowOpen = true;
|
||||
|
||||
// Consume the quoted body; the opening quote has been read.
|
||||
ReadQuotedField(reader, field, ref line, ref col);
|
||||
|
||||
// A closing quote must be followed by the delimiter, a newline, or EOF.
|
||||
var next = reader.Peek();
|
||||
if (next == -1)
|
||||
{
|
||||
row.Add(field.ToString());
|
||||
field.Clear();
|
||||
yield return row.ToArray();
|
||||
row.Clear();
|
||||
rowOpen = false;
|
||||
yield break;
|
||||
}
|
||||
|
||||
var nc = (char)next;
|
||||
if (nc == delimiter)
|
||||
{
|
||||
reader.Read();
|
||||
col++;
|
||||
row.Add(field.ToString());
|
||||
field.Clear();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (nc is Cr or Lf)
|
||||
{
|
||||
row.Add(field.ToString());
|
||||
field.Clear();
|
||||
ConsumeNewline(reader, ref line, ref col);
|
||||
yield return row.ToArray();
|
||||
row.Clear();
|
||||
rowOpen = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new FormatException(
|
||||
$"Unexpected character '{nc}' after closing quote at line {line}, column {col + 1}. " +
|
||||
"A quoted field must be followed by a delimiter, a newline, or end-of-input.");
|
||||
}
|
||||
|
||||
if (c == delimiter)
|
||||
{
|
||||
row.Add(field.ToString());
|
||||
field.Clear();
|
||||
rowOpen = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (c is Cr or Lf)
|
||||
{
|
||||
row.Add(field.ToString());
|
||||
field.Clear();
|
||||
if (c == Cr && reader.Peek() == Lf)
|
||||
{
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
line++;
|
||||
col = 0;
|
||||
yield return row.ToArray();
|
||||
row.Clear();
|
||||
rowOpen = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
field.Append(c);
|
||||
rowOpen = true;
|
||||
}
|
||||
|
||||
// EOF: emit a trailing row only if content is pending. A clean terminator already cleared rowOpen.
|
||||
if (rowOpen || field.Length != 0 || row.Count != 0)
|
||||
{
|
||||
row.Add(field.ToString());
|
||||
yield return row.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the body of a quoted field into <paramref name="field"/>. The opening quote has already
|
||||
/// been consumed. On return the reader sits immediately after the closing quote.
|
||||
/// </summary>
|
||||
private static void ReadQuotedField(TextReader reader, StringBuilder field, ref int line, ref int col)
|
||||
{
|
||||
var openLine = line;
|
||||
var openCol = col;
|
||||
|
||||
int read;
|
||||
while ((read = reader.Read()) != -1)
|
||||
{
|
||||
var c = (char)read;
|
||||
col++;
|
||||
|
||||
if (c == Quote)
|
||||
{
|
||||
if (reader.Peek() == Quote)
|
||||
{
|
||||
reader.Read();
|
||||
col++;
|
||||
field.Append(Quote);
|
||||
continue;
|
||||
}
|
||||
|
||||
return; // closing quote
|
||||
}
|
||||
|
||||
if (c == Lf)
|
||||
{
|
||||
line++;
|
||||
col = 0;
|
||||
}
|
||||
|
||||
field.Append(c);
|
||||
}
|
||||
|
||||
throw new FormatException(
|
||||
$"Unterminated quoted field opened at line {openLine}, column {openCol}: reached end-of-input " +
|
||||
"before the closing double-quote.");
|
||||
}
|
||||
|
||||
/// <summary>Consumes a newline (CRLF, CR, or LF) whose first character has been peeked but not read.</summary>
|
||||
private static void ConsumeNewline(TextReader reader, ref int line, ref int col)
|
||||
{
|
||||
var first = reader.Read();
|
||||
if (first == Cr && reader.Peek() == Lf)
|
||||
{
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
line++;
|
||||
col = 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Header-aware convenience: parses <paramref name="text"/> and maps every subsequent row onto the
|
||||
/// first (header) row's field names. Thin wrapper over <see cref="Parse(string,char)"/>.
|
||||
/// </summary>
|
||||
/// <param name="text">The CSV document; the first row is treated as the header.</param>
|
||||
/// <param name="delimiter">The field separator (default comma).</param>
|
||||
/// <returns>
|
||||
/// One dictionary per data row, keyed by header name (ordinal, case-sensitive). A row shorter than
|
||||
/// the header maps only the columns present; a column beyond the header's width is keyed by its
|
||||
/// 0-based index rendered as a string. An empty document yields zero rows.
|
||||
/// </returns>
|
||||
/// <exception cref="System.FormatException">The input violates the strict RFC 4180 grammar, or the header contains a duplicate column name.</exception>
|
||||
public static IReadOnlyList<IReadOnlyDictionary<string, string>> ParseWithHeader(string? text, char delimiter = ',')
|
||||
{
|
||||
var rows = Parse(text, delimiter);
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
return Array.Empty<IReadOnlyDictionary<string, string>>();
|
||||
}
|
||||
|
||||
var header = rows[0];
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var name in header)
|
||||
{
|
||||
if (!seen.Add(name))
|
||||
{
|
||||
throw new FormatException($"Duplicate header column name '{name}'.");
|
||||
}
|
||||
}
|
||||
|
||||
var result = new List<IReadOnlyDictionary<string, string>>(rows.Count - 1);
|
||||
for (var i = 1; i < rows.Count; i++)
|
||||
{
|
||||
var cells = rows[i];
|
||||
var map = new Dictionary<string, string>(cells.Length, StringComparer.Ordinal);
|
||||
for (var c = 0; c < cells.Length; c++)
|
||||
{
|
||||
var key = c < header.Length ? header[c] : c.ToString(CultureInfo.InvariantCulture);
|
||||
map[key] = cells[c];
|
||||
}
|
||||
|
||||
result.Add(map);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.Csv;
|
||||
|
||||
/// <summary>
|
||||
/// A pure RFC 4180 CSV writer with no file/stream I/O of its own — it renders rows of fields to a
|
||||
/// <see cref="string"/> or a <see cref="TextWriter"/>. The inverse of <see cref="CsvParser"/>:
|
||||
/// <c>CsvParser.Parse(CsvWriter.WriteToString(rows))</c> reproduces <c>rows</c> exactly for any field
|
||||
/// content.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para><b>Quote-on-demand.</b> A field is wrapped in double-quotes only when it must be — i.e. when
|
||||
/// it contains the delimiter, a double-quote, CR, or LF — and internal double-quotes are doubled
|
||||
/// (<c>"</c> → <c>""</c>). Fields that need no quoting are emitted verbatim, so ordinary values stay
|
||||
/// human-readable. Pass <c>quoteAllFields: true</c> to force every field quoted.</para>
|
||||
/// <para><b>Newline.</b> The record terminator defaults to CRLF (<c>\r\n</c>) per RFC 4180 and is
|
||||
/// configurable. No terminator is written after the final row (matching the parser's
|
||||
/// no-phantom-trailing-row contract, so a round-trip is exact).</para>
|
||||
/// </remarks>
|
||||
public static class CsvWriter
|
||||
{
|
||||
private const char Quote = '"';
|
||||
private const char Cr = '\r';
|
||||
private const char Lf = '\n';
|
||||
|
||||
/// <summary>The RFC 4180 record terminator, <c>"\r\n"</c>. The default <c>newline</c> for every write.</summary>
|
||||
public const string Crlf = "\r\n";
|
||||
|
||||
/// <summary>
|
||||
/// Renders <paramref name="rows"/> to a CSV string.
|
||||
/// </summary>
|
||||
/// <param name="rows">The rows to write; each inner sequence is one record's fields. A <c>null</c> field is written as empty.</param>
|
||||
/// <param name="delimiter">The field separator (default comma).</param>
|
||||
/// <param name="newline">The record terminator between rows (default CRLF). Not appended after the last row.</param>
|
||||
/// <param name="quoteAllFields">When <c>true</c>, every field is quoted regardless of content.</param>
|
||||
/// <returns>The CSV text. An empty <paramref name="rows"/> yields the empty string.</returns>
|
||||
/// <exception cref="System.ArgumentNullException"><paramref name="rows"/> is <c>null</c>.</exception>
|
||||
/// <exception cref="System.ArgumentException"><paramref name="delimiter"/> is a quote, CR, or LF.</exception>
|
||||
public static string WriteToString(
|
||||
IEnumerable<IEnumerable<string?>> rows,
|
||||
char delimiter = ',',
|
||||
string newline = Crlf,
|
||||
bool quoteAllFields = false)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
using var writer = new StringWriter(sb);
|
||||
Write(writer, rows, delimiter, newline, quoteAllFields);
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes <paramref name="rows"/> to <paramref name="writer"/>. The writer is not disposed or
|
||||
/// flushed by this method.
|
||||
/// </summary>
|
||||
/// <param name="writer">The destination.</param>
|
||||
/// <param name="rows">The rows to write; each inner sequence is one record's fields. A <c>null</c> field is written as empty.</param>
|
||||
/// <param name="delimiter">The field separator (default comma).</param>
|
||||
/// <param name="newline">The record terminator between rows (default CRLF). Not appended after the last row.</param>
|
||||
/// <param name="quoteAllFields">When <c>true</c>, every field is quoted regardless of content.</param>
|
||||
/// <exception cref="System.ArgumentNullException"><paramref name="writer"/> or <paramref name="rows"/> is <c>null</c>.</exception>
|
||||
/// <exception cref="System.ArgumentException"><paramref name="delimiter"/> is a quote, CR, or LF.</exception>
|
||||
public static void Write(
|
||||
TextWriter writer,
|
||||
IEnumerable<IEnumerable<string?>> rows,
|
||||
char delimiter = ',',
|
||||
string newline = Crlf,
|
||||
bool quoteAllFields = false)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(writer);
|
||||
ArgumentNullException.ThrowIfNull(rows);
|
||||
if (delimiter is Quote or Cr or Lf)
|
||||
{
|
||||
throw new ArgumentException("Delimiter must not be a double-quote, CR, or LF.", nameof(delimiter));
|
||||
}
|
||||
|
||||
var firstRow = true;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (!firstRow)
|
||||
{
|
||||
writer.Write(newline);
|
||||
}
|
||||
|
||||
firstRow = false;
|
||||
|
||||
WriteRow(writer, row, delimiter, quoteAllFields);
|
||||
}
|
||||
}
|
||||
|
||||
private static void WriteRow(TextWriter writer, IEnumerable<string?> row, char delimiter, bool quoteAllFields)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(row);
|
||||
|
||||
var firstField = true;
|
||||
foreach (var field in row)
|
||||
{
|
||||
if (!firstField)
|
||||
{
|
||||
writer.Write(delimiter);
|
||||
}
|
||||
|
||||
firstField = false;
|
||||
|
||||
WriteField(writer, field ?? string.Empty, delimiter, quoteAllFields);
|
||||
}
|
||||
}
|
||||
|
||||
private static void WriteField(TextWriter writer, string field, char delimiter, bool quoteAllFields)
|
||||
{
|
||||
if (!quoteAllFields && !NeedsQuoting(field, delimiter))
|
||||
{
|
||||
writer.Write(field);
|
||||
return;
|
||||
}
|
||||
|
||||
writer.Write(Quote);
|
||||
foreach (var ch in field)
|
||||
{
|
||||
if (ch == Quote)
|
||||
{
|
||||
writer.Write(Quote); // double an internal quote
|
||||
}
|
||||
|
||||
writer.Write(ch);
|
||||
}
|
||||
|
||||
writer.Write(Quote);
|
||||
}
|
||||
|
||||
private static bool NeedsQuoting(string field, char delimiter)
|
||||
{
|
||||
foreach (var ch in field)
|
||||
{
|
||||
if (ch == delimiter || ch == Quote || ch == Cr || ch == Lf)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user