fix: address lexer code review findings (newline handling, emit cleanup, null guard)
This commit is contained in:
@@ -61,6 +61,7 @@ public sealed class NatsConfLexer
|
|||||||
|
|
||||||
public static IReadOnlyList<Token> Tokenize(string input)
|
public static IReadOnlyList<Token> Tokenize(string input)
|
||||||
{
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(input);
|
||||||
var lx = new NatsConfLexer(input);
|
var lx = new NatsConfLexer(input);
|
||||||
LexState? state = LexTop;
|
LexState? state = LexTop;
|
||||||
while (state is not null)
|
while (state is not null)
|
||||||
@@ -85,7 +86,17 @@ public sealed class NatsConfLexer
|
|||||||
|
|
||||||
private void Emit(TokenType type)
|
private void Emit(TokenType type)
|
||||||
{
|
{
|
||||||
var val = string.Concat(_stringParts) + _input[_start.._pos];
|
string val;
|
||||||
|
if (_stringParts.Count > 0)
|
||||||
|
{
|
||||||
|
val = string.Concat(_stringParts) + _input[_start.._pos];
|
||||||
|
_stringParts.Clear();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
val = _input[_start.._pos];
|
||||||
|
}
|
||||||
|
|
||||||
var pos = _pos - _ilstart - val.Length;
|
var pos = _pos - _ilstart - val.Length;
|
||||||
_items.Add(new Token(type, val, _line, pos));
|
_items.Add(new Token(type, val, _line, pos));
|
||||||
_start = _pos;
|
_start = _pos;
|
||||||
@@ -660,7 +671,7 @@ public sealed class NatsConfLexer
|
|||||||
private static LexState? LexKeyEnd(NatsConfLexer lx)
|
private static LexState? LexKeyEnd(NatsConfLexer lx)
|
||||||
{
|
{
|
||||||
var r = lx.Next();
|
var r = lx.Next();
|
||||||
if (char.IsWhiteSpace(r) && !IsNL(r))
|
if (char.IsWhiteSpace(r))
|
||||||
{
|
{
|
||||||
return LexSkip(lx, LexKeyEnd);
|
return LexSkip(lx, LexKeyEnd);
|
||||||
}
|
}
|
||||||
@@ -961,7 +972,7 @@ public sealed class NatsConfLexer
|
|||||||
private static LexState? LexMapKeyEnd(NatsConfLexer lx)
|
private static LexState? LexMapKeyEnd(NatsConfLexer lx)
|
||||||
{
|
{
|
||||||
var r = lx.Next();
|
var r = lx.Next();
|
||||||
if (char.IsWhiteSpace(r) && !IsNL(r))
|
if (char.IsWhiteSpace(r))
|
||||||
{
|
{
|
||||||
return LexSkip(lx, LexMapKeyEnd);
|
return LexSkip(lx, LexMapKeyEnd);
|
||||||
}
|
}
|
||||||
@@ -1239,13 +1250,13 @@ public sealed class NatsConfLexer
|
|||||||
private static LexState? LexStringBinary(NatsConfLexer lx)
|
private static LexState? LexStringBinary(NatsConfLexer lx)
|
||||||
{
|
{
|
||||||
var r1 = lx.Next();
|
var r1 = lx.Next();
|
||||||
if (IsNL(r1))
|
if (IsNL(r1) || r1 == Eof)
|
||||||
{
|
{
|
||||||
return lx.Errorf("Expected two hexadecimal digits after '\\x', but hit end of line");
|
return lx.Errorf("Expected two hexadecimal digits after '\\x', but hit end of line");
|
||||||
}
|
}
|
||||||
|
|
||||||
var r2 = lx.Next();
|
var r2 = lx.Next();
|
||||||
if (IsNL(r2))
|
if (IsNL(r2) || r2 == Eof)
|
||||||
{
|
{
|
||||||
return lx.Errorf("Expected two hexadecimal digits after '\\x', but hit end of line");
|
return lx.Errorf("Expected two hexadecimal digits after '\\x', but hit end of line");
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user