Skip to content

Commit

Permalink
Add support for spread operator
Browse files Browse the repository at this point in the history
  • Loading branch information
@VincentLanglet
VincentLanglet committed Jul 21, 2023
1 parent 271faaa commit ec651f2
Show file tree
Hide file tree
Showing 5 changed files with 89 additions and 27 deletions.
18 changes: 9 additions & 9 deletions src/Sniff/AbstractSniff.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,10 @@ public function fixFile(array $stream, FixerInterface $fixer): void
abstract protected function process(int $tokenPosition, array $tokens): void;

/**
* @param int|int[] $type
* @param string|string[] $value
* @param int|string|array<int|string> $type
* @param string|string[] $value
*/
protected function isTokenMatching(Token $token, int|array $type, string|array $value = []): bool
protected function isTokenMatching(Token $token, int|string|array $type, string|array $value = []): bool
{
if (!\is_array($type)) {
$type = [$type];
Expand All @@ -58,10 +58,10 @@ protected function isTokenMatching(Token $token, int|array $type, string|array $
}

/**
* @param int|int[] $type
* @param array<int, Token> $tokens
* @param int|string|array<int|string> $type
* @param array<int, Token> $tokens
*/
protected function findNext(int|array $type, array $tokens, int $start, bool $exclude = false): int|false
protected function findNext(int|string|array $type, array $tokens, int $start, bool $exclude = false): int|false
{
$i = 0;

Expand All @@ -80,10 +80,10 @@ protected function findNext(int|array $type, array $tokens, int $start, bool $ex
}

/**
* @param int|int[] $type
* @param array<int, Token> $tokens
* @param int|string|array<int|string> $type
* @param array<int, Token> $tokens
*/
protected function findPrevious(int|array $type, array $tokens, int $start, bool $exclude = false): int|false
protected function findPrevious(int|string|array $type, array $tokens, int $start, bool $exclude = false): int|false
{
$i = 0;

Expand Down
29 changes: 15 additions & 14 deletions src/Token/Token.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,20 @@ final class Token
public const INTERPOLATION_START_TYPE = 10;
public const INTERPOLATION_END_TYPE = 11;
public const ARROW_TYPE = 12;
public const SPREAD_TYPE = 13;
// New constants
public const DQ_STRING_START_TYPE = 13;
public const DQ_STRING_END_TYPE = 14;
public const BLOCK_TAG_TYPE = 15;
public const WHITESPACE_TYPE = 16;
public const TAB_TYPE = 17;
public const EOL_TYPE = 18;
public const COMMENT_START_TYPE = 19;
public const COMMENT_TEXT_TYPE = 20;
public const COMMENT_WHITESPACE_TYPE = 21;
public const COMMENT_TAB_TYPE = 22;
public const COMMENT_EOL_TYPE = 23;
public const COMMENT_END_TYPE = 24;
public const DQ_STRING_START_TYPE = 'DQ_STRING_START_TYPE';
public const DQ_STRING_END_TYPE = 'DQ_STRING_END_TYPE';
public const BLOCK_TAG_TYPE = 'BLOCK_TAG_TYPE';
public const WHITESPACE_TYPE = 'WHITESPACE_TYPE';
public const TAB_TYPE = 'TAB_TYPE';
public const EOL_TYPE = 'EOL_TYPE';
public const COMMENT_START_TYPE = 'COMMENT_START_TYPE';
public const COMMENT_TEXT_TYPE = 'COMMENT_TEXT_TYPE';
public const COMMENT_WHITESPACE_TYPE = 'COMMENT_WHITESPACE_TYPE';
public const COMMENT_TAB_TYPE = 'COMMENT_TAB_TYPE';
public const COMMENT_EOL_TYPE = 'COMMENT_EOL_TYPE';
public const COMMENT_END_TYPE = 'COMMENT_END_TYPE';

public const WHITESPACE_TOKENS = [
self::WHITESPACE_TYPE => self::WHITESPACE_TYPE,
Expand All @@ -62,7 +63,7 @@ final class Token
];

public function __construct(
private int $type,
private int|string $type,
private int $line,
private int $position,
private string $filename,
Expand All @@ -71,7 +72,7 @@ public function __construct(
) {
}

public function getType(): int
public function getType(): int|string
{
return $this->type;
}
Expand Down
10 changes: 9 additions & 1 deletion src/Token/Tokenizer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ private function moveCurrentExpressionStarter(): void
$this->currentExpressionStarter++;
}

private function pushToken(int $type, string $value = '', ?Token $relatedToken = null): Token
private function pushToken(int|string $type, string $value = '', ?Token $relatedToken = null): Token
{
$token = new Token(
$type,
Expand All @@ -291,13 +291,16 @@ private function lexExpression(): void
{
$currentCode = $this->code[$this->cursor];
$nextToken = $this->code[$this->cursor + 1] ?? null;
$next2Token = $this->code[$this->cursor + 2] ?? null;

if (1 === preg_match('/\t/', $currentCode)) {
$this->lexTab();
} elseif (' ' === $currentCode) {
$this->lexWhitespace();
} elseif (1 === preg_match("/\r\n?|\n/", $currentCode)) {
$this->lexEOL();
} elseif ('.' === $currentCode && '.' === $nextToken && '.' === $next2Token) {
$this->lexSpread();
} elseif ('=' === $currentCode && '>' === $nextToken) {
$this->lexArrowFunction();
} elseif (1 === preg_match($this->operatorRegex, $this->code, $match, 0, $this->cursor)) {
Expand Down Expand Up @@ -528,6 +531,11 @@ private function lexEOL(): void
$this->lastEOL = $this->cursor;
}

private function lexSpread(): void
{
$this->pushToken(Token::SPREAD_TYPE, '...');
}

private function lexArrowFunction(): void
{
$this->pushToken(Token::ARROW_TYPE, '=>');
Expand Down
2 changes: 2 additions & 0 deletions tests/Token/Tokenizer/Fixtures/test13.twig
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{{ { a: "a", ...{ b: "b" } } }}
{{ [1, 2, ...[3, 4]] }}
57 changes: 54 additions & 3 deletions tests/Token/Tokenizer/TokenizerTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public function testTokenizeWithCustomOperators(): void
}

/**
* @param array<int, int> $expectedTokenTypes
* @param array<int, int|string> $expectedTokenTypes
*
* @dataProvider tokenizeDataProvider
*/
Expand All @@ -91,12 +91,12 @@ public function testTokenizeTypes(string $filePath, array $expectedTokenTypes):
static::fail($diff);
}

$tokenTypes = array_map(static fn (Token $token): int => $token->getType(), $tokens);
$tokenTypes = array_map(static fn (Token $token): int|string => $token->getType(), $tokens);
static::assertSame($expectedTokenTypes, $tokenTypes);
}

/**
* @return iterable<array-key, array{string, array<int, int>}>
* @return iterable<array-key, array{string, array<int, int|string>}>
*/
public static function tokenizeDataProvider(): iterable
{
Expand Down Expand Up @@ -536,6 +536,57 @@ public static function tokenizeDataProvider(): iterable
58 => Token::EOF_TYPE,
],
];

yield [
__DIR__.'/Fixtures/test13.twig',
[
0 => Token::VAR_START_TYPE,
1 => Token::WHITESPACE_TYPE,
2 => Token::PUNCTUATION_TYPE,
3 => Token::WHITESPACE_TYPE,
4 => Token::NAME_TYPE,
5 => Token::PUNCTUATION_TYPE,
6 => Token::WHITESPACE_TYPE,
7 => Token::STRING_TYPE,
8 => Token::PUNCTUATION_TYPE,
9 => Token::WHITESPACE_TYPE,
10 => Token::SPREAD_TYPE,
11 => Token::PUNCTUATION_TYPE,
12 => Token::WHITESPACE_TYPE,
13 => Token::NAME_TYPE,
14 => Token::PUNCTUATION_TYPE,
15 => Token::WHITESPACE_TYPE,
16 => Token::STRING_TYPE,
17 => Token::WHITESPACE_TYPE,
18 => Token::PUNCTUATION_TYPE,
19 => Token::WHITESPACE_TYPE,
20 => Token::PUNCTUATION_TYPE,
21 => Token::WHITESPACE_TYPE,
22 => Token::VAR_END_TYPE,
23 => Token::EOL_TYPE,
24 => Token::VAR_START_TYPE,
25 => Token::WHITESPACE_TYPE,
26 => Token::PUNCTUATION_TYPE,
27 => Token::NUMBER_TYPE,
28 => Token::PUNCTUATION_TYPE,
29 => Token::WHITESPACE_TYPE,
30 => Token::NUMBER_TYPE,
31 => Token::PUNCTUATION_TYPE,
32 => Token::WHITESPACE_TYPE,
33 => Token::SPREAD_TYPE,
34 => Token::PUNCTUATION_TYPE,
35 => Token::NUMBER_TYPE,
36 => Token::PUNCTUATION_TYPE,
37 => Token::WHITESPACE_TYPE,
38 => Token::NUMBER_TYPE,
39 => Token::PUNCTUATION_TYPE,
40 => Token::PUNCTUATION_TYPE,
41 => Token::WHITESPACE_TYPE,
42 => Token::VAR_END_TYPE,
43 => Token::EOL_TYPE,
44 => Token::EOF_TYPE,
],
];
}

/**
Expand Down

0 comments on commit ec651f2

Please sign in to comment.