Skip to content

Commit

Permalink
PHP 8.0 | Tokenizer/PHP: backfill the T_MATCH tokenization
Browse files Browse the repository at this point in the history 
PHP 8.0 introduces a new type of control structure: match expressions.
> A match expression is similar to a `switch` control structure but with safer semantics and the ability to return values.

Ref: https://wiki.php.net/rfc/match_expression_v2

This commit adds initial support for match expressions to PHPCS.

* In PHP < 8: Retokenizes `T_STRING` tokens containing the `match` keyword to `T_MATCH` when they are in actual fact match expressions.
* In PHP 8: Retokenizes `T_MATCH` tokens to `T_STRING` when the `match` keyword is used outside the context of a match expression, like in a method declaration or call.
* Ensures that the `match` keyword for match expressions will be recognized as a scope owner and that the appropriate `scope_*` array indexes are set for the curly braces belonging to the match expression, as well as that the `match` condition is added to the tokens within the match expression.

Note: in contrast to `switch` control structures, "cases" in a match expression will not be recognized as scopes and no `scope_owner`, `scope_opener` or `scope_closer` array indexes will be set for the individual cases in a match expression.
This also applies to the `default` case when used in a match expression.

Includes extensive unit tests.
Note: at this point, not all unit tests will pass, this will be fixed in follow-on commits.
  • Loading branch information
@jrfnl
jrfnl committed Feb 22, 2021
1 parent 966ae7c commit 08a946f
Show file tree
Hide file tree
Showing 4 changed files with 949 additions and 0 deletions.
6 changes: 6 additions & 0 deletions package.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
<file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
<file baseinstalldir="" name="BackfillMatchTokenTest.inc" role="test" />
<file baseinstalldir="" name="BackfillMatchTokenTest.php" role="test" />
<file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
<file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
<file baseinstalldir="" name="BitwiseOrTest.inc" role="test" />
Expand Down Expand Up @@ -2100,6 +2102,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.inc" name="tests/Core/Tokenizer/BackfillMatchTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
Expand Down Expand Up @@ -2176,6 +2180,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.inc" name="tests/Core/Tokenizer/BackfillMatchTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
Expand Down
119 changes: 119 additions & 0 deletions src/Tokenizers/PHP.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,13 @@ class PHP extends Tokenizer
T_SWITCH => T_SWITCH,
],
],
T_MATCH => [
'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
'strict' => true,
'shared' => false,
'with' => [],
],
T_START_HEREDOC => [
'start' => [T_START_HEREDOC => T_START_HEREDOC],
'end' => [T_END_HEREDOC => T_END_HEREDOC],
Expand Down Expand Up @@ -365,6 +372,7 @@ class PHP extends Tokenizer
T_LOGICAL_AND => 3,
T_LOGICAL_OR => 2,
T_LOGICAL_XOR => 3,
T_MATCH => 5,
T_METHOD_C => 10,
T_MINUS_EQUAL => 2,
T_POW_EQUAL => 3,
Expand Down Expand Up @@ -1254,6 +1262,87 @@ protected function tokenize($string)
continue;
}//end if

/*
Backfill the T_MATCH token for PHP versions < 8.0 and
do initial correction for non-match expression T_MATCH tokens
to T_STRING for PHP >= 8.0.
A final check for non-match expression T_MATCH tokens is done
in PHP::processAdditional().
*/

if ($tokenIsArray === true
&& (($token[0] === T_STRING
&& strtolower($token[1]) === 'match')
|| $token[0] === T_MATCH)
) {
$isMatch = false;
for ($x = ($stackPtr + 1); $x < $numTokens; $x++) {
if (isset($tokens[$x][0], Util\Tokens::$emptyTokens[$tokens[$x][0]]) === true) {
continue;
}

if ($tokens[$x] !== '(') {
// This is not a match expression.
break;
}

// Next was an open parenthesis, now check what is before the match keyword.
for ($y = ($stackPtr - 1); $y >= 0; $y--) {
if (isset(Util\Tokens::$emptyTokens[$tokens[$y][0]]) === true) {
continue;
}

if (is_array($tokens[$y]) === true
&& ($tokens[$y][0] === T_PAAMAYIM_NEKUDOTAYIM
|| $tokens[$y][0] === T_OBJECT_OPERATOR
|| $tokens[$y][0] === T_NS_SEPARATOR
|| $tokens[$y][0] === T_NEW
|| $tokens[$y][0] === T_FUNCTION
|| $tokens[$y][0] === T_CLASS
|| $tokens[$y][0] === T_INTERFACE
|| $tokens[$y][0] === T_TRAIT
|| $tokens[$y][0] === T_NAMESPACE
|| $tokens[$y][0] === T_CONST)
) {
// This is not a match expression.
break 2;
}

$isMatch = true;
break 2;
}//end for
}//end for

if ($isMatch === true && $token[0] === T_STRING) {
$newToken = [];
$newToken['code'] = T_MATCH;
$newToken['type'] = 'T_MATCH';
$newToken['content'] = $token[1];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token $stackPtr changed from T_STRING to T_MATCH".PHP_EOL;
}

$finalTokens[$newStackPtr] = $newToken;
$newStackPtr++;
continue;
} else if ($isMatch === false && $token[0] === T_MATCH) {
// PHP 8.0, match keyword, but not a match expression.
$newToken = [];
$newToken['code'] = T_STRING;
$newToken['type'] = 'T_STRING';
$newToken['content'] = $token[1];

if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token $stackPtr changed from T_MATCH to T_STRING".PHP_EOL;
}

$finalTokens[$newStackPtr] = $newToken;
$newStackPtr++;
continue;
}//end if
}//end if

/*
Convert ? to T_NULLABLE OR T_INLINE_THEN
*/
Expand Down Expand Up @@ -2265,6 +2354,36 @@ protected function processAdditional()
}
}

continue;
} else if ($this->tokens[$i]['code'] === T_MATCH) {
if (isset($this->tokens[$i]['scope_opener'], $this->tokens[$i]['scope_closer']) === false) {
// Not a match expression after all.
$this->tokens[$i]['code'] = T_STRING;
$this->tokens[$i]['type'] = 'T_STRING';

if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token $i changed from T_MATCH to T_STRING".PHP_EOL;
}

if (isset($this->tokens[$i]['parenthesis_opener'], $this->tokens[$i]['parenthesis_closer']) === true) {
$opener = $this->tokens[$i]['parenthesis_opener'];
$closer = $this->tokens[$i]['parenthesis_closer'];
unset(
$this->tokens[$opener]['parenthesis_owner'],
$this->tokens[$closer]['parenthesis_owner']
);
unset(
$this->tokens[$i]['parenthesis_opener'],
$this->tokens[$i]['parenthesis_closer'],
$this->tokens[$i]['parenthesis_owner']
);

if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* cleaned parenthesis of token $i *".PHP_EOL;
}
}
}//end if

continue;
} else if ($this->tokens[$i]['code'] === T_BITWISE_OR) {
/*
Expand Down
Loading

0 comments on commit 08a946f

Please sign in to comment.