Skip to content

Commit

Permalink
Backport underscores in integer literals
Browse files Browse the repository at this point in the history
  • Loading branch information
@slozier
slozier committed Dec 25, 2024
1 parent 0ad2080 commit e16b5fe
Show file tree
Hide file tree
Showing 5 changed files with 247 additions and 325 deletions.
78 changes: 42 additions & 36 deletions Src/IronPython/Compiler/Tokenizer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -735,11 +735,16 @@ private Token ReadNumber(int start) {
}
isPrefix0 = true;

while (NextChar('0')) { } // skip leading zeroes
// skip leading zeroes
while (true) {
NextChar('_');
if (!NextChar('0')) break;
}
}

bool isFirstChar = true;
while (true) {
NextChar('_');
int ch = NextChar();

switch (ch) {
Expand All @@ -755,7 +760,7 @@ private Token ReadNumber(int start) {
MarkTokenEnd();

// TODO: parse in place
return new ConstantValueToken(ParseInteger(GetTokenString(), 10));
return new ConstantValueToken(ParseInteger(GetTokenSpan(), 10));

case 'j':
case 'J':
Expand Down Expand Up @@ -784,7 +789,7 @@ private Token ReadNumber(int start) {
}

// TODO: parse in place
return new ConstantValueToken(ParseInteger(GetTokenString(), 10));
return new ConstantValueToken(ParseInteger(GetTokenSpan(), 10));
}
isFirstChar = false;
}
Expand All @@ -795,8 +800,9 @@ private Token ReadBinaryNumber() {
int iVal = 0;
bool useBigInt = false;
BigInteger bigInt = BigInteger.Zero;
bool first = true;
bool isFirstChar = true;
while (true) {
NextChar('_');
int ch = NextChar();
switch (ch) {
case '0':
Expand All @@ -812,7 +818,7 @@ private Token ReadBinaryNumber() {
bigInt = (BigInteger)iVal;
}

if (bits >= 32) {
if (useBigInt) {
bigInt = (bigInt << 1) | (ch - '0');
} else {
iVal = iVal << 1 | (ch - '0');
Expand All @@ -822,22 +828,21 @@ private Token ReadBinaryNumber() {
BufferBack();
MarkTokenEnd();

if (first) {
ReportSyntaxError(
new SourceSpan(new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1),
BufferTokenEnd),
Resources.InvalidToken, ErrorCodes.SyntaxError);
if (isFirstChar) {
var errorStart = new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1);
ReportSyntaxError(new SourceSpan(errorStart, BufferTokenEnd), Resources.InvalidToken, ErrorCodes.SyntaxError);
}

return new ConstantValueToken(useBigInt ? bigInt : (object)iVal);
}
first = false;
isFirstChar = false;
}
}

private Token ReadOctalNumber() {
bool first = true;
bool isFirstChar = true;
while (true) {
NextChar('_');
int ch = NextChar();

switch (ch) {
Expand All @@ -855,23 +860,24 @@ private Token ReadOctalNumber() {
BufferBack();
MarkTokenEnd();

if (first) {
ReportSyntaxError(
new SourceSpan(new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1),
BufferTokenEnd),
Resources.InvalidToken, ErrorCodes.SyntaxError);
if (isFirstChar) {
var errorStart = new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1);
ReportSyntaxError(new SourceSpan(errorStart, BufferTokenEnd), Resources.InvalidToken, ErrorCodes.SyntaxError);
}

// TODO: parse in place
return new ConstantValueToken(ParseInteger(GetTokenSubstring(2), 8));
var span = GetTokenSpan().Slice(2);
if (!span.IsEmpty && span[0] == '_') span = span.Slice(1);
return new ConstantValueToken(ParseInteger(span, 8));
}
first = false;
isFirstChar = false;
}
}

private Token ReadHexNumber() {
bool first = true;
bool isFirstChar = true;
while (true) {
NextChar('_');
int ch = NextChar();

switch (ch) {
Expand Down Expand Up @@ -903,17 +909,17 @@ private Token ReadHexNumber() {
BufferBack();
MarkTokenEnd();

if (first) {
ReportSyntaxError(
new SourceSpan(new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1),
BufferTokenEnd),
Resources.InvalidToken, ErrorCodes.SyntaxError);
if (isFirstChar) {
var errorStart = new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1);
ReportSyntaxError(new SourceSpan(errorStart, BufferTokenEnd), Resources.InvalidToken, ErrorCodes.SyntaxError);
}

// TODO: parse in place
return new ConstantValueToken(ParseInteger(GetTokenSubstring(2), 16));
var span = GetTokenSpan().Slice(2);
if (!span.IsEmpty && span[0] == '_') span = span.Slice(1);
return new ConstantValueToken(ParseInteger(span, 16));
}
first = false;
isFirstChar = false;
}
}

Expand Down Expand Up @@ -1431,10 +1437,8 @@ private void SetIndent(int spaces, StringBuilder chars) {
current = DoDedent(spaces, current);

if (spaces != current) {
ReportSyntaxError(
new SourceSpan(new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1),
BufferTokenEnd),
Resources.IndentationMismatch, ErrorCodes.IndentationError);
var errorStart = new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1);
ReportSyntaxError(new SourceSpan(errorStart, BufferTokenEnd), Resources.IndentationMismatch, ErrorCodes.IndentationError);
}
}
}
Expand All @@ -1448,12 +1452,11 @@ private int DoDedent(int spaces, int current) {
return current;
}

private object ParseInteger(string s, int radix) {
try {
return LiteralParser.ParseInteger(s, radix);
} catch (ArgumentException e) {
ReportSyntaxError(BufferTokenSpan, e.Message, ErrorCodes.SyntaxError);
private object ParseInteger(ReadOnlySpan<char> s, int radix) {
if (LiteralParser.TryParseIntegerSign(s, radix, out object result)) {
return result;
}
ReportSyntaxError(BufferTokenSpan, "invalid token", ErrorCodes.SyntaxError);
return ScriptingRuntimeHelpers.Int32ToObject(0);
}

Expand Down Expand Up @@ -1673,6 +1676,9 @@ private string GetTokenSubstring(int offset, int length) {
return new String(_buffer, _start + offset, length);
}

private ReadOnlySpan<char> GetTokenSpan()
=> _buffer.AsSpan(_start, _tokenEnd - _start);

[Conditional("DEBUG")]
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic")]
private void CheckInvariants() {
Expand Down
Loading

0 comments on commit e16b5fe

Please sign in to comment.