Skip to content

Commit

Permalink
Merge branch 'feature/tokenizer_attributes' of https://github.com/ale…
Browse files Browse the repository at this point in the history 
…kitto/PHP_CodeSniffer
  • Loading branch information
@gsherwood
gsherwood committed Mar 29, 2021
2 parents fe68f36 + 6367e71 commit 013943d
Show file tree
Hide file tree
Showing 9 changed files with 852 additions and 0 deletions.
6 changes: 6 additions & 0 deletions package.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<dir name="Tokenizer">
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.inc" role="test" />
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
<file baseinstalldir="" name="AttributesTest.inc" role="test" />
<file baseinstalldir="" name="AttributesTest.php" role="test" />
<file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
<file baseinstalldir="" name="BackfillMatchTokenTest.inc" role="test" />
Expand Down Expand Up @@ -2139,6 +2141,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
Expand Down Expand Up @@ -2223,6 +2227,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
Expand Down
1 change: 1 addition & 0 deletions src/Files/File.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1813,6 +1813,7 @@ public function getMemberProperties($stackPtr)
T_SEMICOLON,
T_OPEN_CURLY_BRACKET,
T_CLOSE_CURLY_BRACKET,
T_ATTRIBUTE_END,
],
($stackPtr - 1)
);
Expand Down
171 changes: 171 additions & 0 deletions src/Tokenizers/PHP.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -903,6 +903,50 @@ protected function tokenize($string)
continue;
}//end if

/*
PHP 8.0 Attributes
*/

if (PHP_VERSION_ID < 80000
&& $token[0] === T_COMMENT
&& strpos($token[1], '#[') === 0
) {
$subTokens = $this->parsePhpAttribute($tokens, $stackPtr);
if ($subTokens !== null) {
array_splice($tokens, $stackPtr, 1, $subTokens);
$numTokens = count($tokens);

$tokenIsArray = true;
$token = $tokens[$stackPtr];
} else {
$token[0] = T_ATTRIBUTE;
}
}

if ($tokenIsArray === true
&& $token[0] === T_ATTRIBUTE
) {
// Go looking for the close bracket.
$bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), ['[', '#['], ']');

$newToken = [];
$newToken['code'] = T_ATTRIBUTE;
$newToken['type'] = 'T_ATTRIBUTE';
$newToken['content'] = '#[';
$finalTokens[$newStackPtr] = $newToken;

$tokens[$bracketCloser] = [];
$tokens[$bracketCloser][0] = T_ATTRIBUTE_END;
$tokens[$bracketCloser][1] = ']';

if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token $bracketCloser changed from T_CLOSE_SQUARE_BRACKET to T_ATTRIBUTE_END".PHP_EOL;
}

$newStackPtr++;
continue;
}//end if

/*
Tokenize the parameter labels for PHP 8.0 named parameters as a special T_PARAM_NAME
token and ensure that the colon after it is always T_COLON.
Expand Down Expand Up @@ -1857,6 +1901,7 @@ function return types. We want to keep the parenthesis map clean,
T_CLASS => true,
T_EXTENDS => true,
T_IMPLEMENTS => true,
T_ATTRIBUTE => true,
T_NEW => true,
T_CONST => true,
T_NS_SEPARATOR => true,
Expand Down Expand Up @@ -2114,6 +2159,8 @@ protected function processAdditional()
echo "\t*** START ADDITIONAL PHP PROCESSING ***".PHP_EOL;
}

$this->createAttributesNestingMap();

$numTokens = count($this->tokens);
for ($i = ($numTokens - 1); $i >= 0; $i--) {
// Check for any unset scope conditions due to alternate IF/ENDIF syntax.
Expand Down Expand Up @@ -3089,4 +3136,128 @@ public static function resolveSimpleToken($token)
}//end resolveSimpleToken()


/**
* Finds a "closer" token (closing parenthesis or square bracket for example)
* Handle parenthesis balancing while searching for closing token
*
* @param array $tokens The list of tokens to iterate searching the closing token (as returned by token_get_all)
* @param int $start The starting position
* @param string|string[] $openerTokens The opening character
* @param string $closerChar The closing character
*
* @return int|null The position of the closing token, if found. NULL otherwise.
*/
private function findCloser(array &$tokens, $start, $openerTokens, $closerChar)
{
$numTokens = count($tokens);
$stack = [0];
$closer = null;
$openerTokens = (array) $openerTokens;

for ($x = $start; $x < $numTokens; $x++) {
if (in_array($tokens[$x], $openerTokens, true) === true
|| (is_array($tokens[$x]) === true && in_array($tokens[$x][1], $openerTokens, true) === true)
) {
$stack[] = $x;
} else if ($tokens[$x] === $closerChar) {
array_pop($stack);
if (empty($stack) === true) {
$closer = $x;
break;
}
}
}

return $closer;

}//end findCloser()


/**
* PHP 8 attributes parser for PHP < 8
* Handles single-line and multiline attributes.
*
* @param array $tokens The original array of tokens (as returned by token_get_all)
* @param int $stackPtr The current position in token array
*
* @return array|null The array of parsed attribute tokens
*/
private function parsePhpAttribute(array &$tokens, $stackPtr)
{

$token = $tokens[$stackPtr];

$commentBody = substr($token[1], 2);
$subTokens = @token_get_all('<?php '.$commentBody);

foreach ($subTokens as $i => $subToken) {
if (is_array($subToken) === true
&& $subToken[0] === T_COMMENT
&& strpos($subToken[1], '#[') === 0
) {
$reparsed = $this->parsePhpAttribute($subTokens, $i);
if ($reparsed !== null) {
array_splice($subTokens, $i, 1, $reparsed);
} else {
$subToken[0] = T_ATTRIBUTE;
}
}
}

array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);

// Go looking for the close bracket.
$bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
if ($bracketCloser === null) {
$bracketCloser = $this->findCloser($tokens, $stackPtr, '[', ']');
if ($bracketCloser === null) {
return null;
}

$subTokens = array_merge($subTokens, array_slice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr)));
array_splice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr));
}

return $subTokens;

}//end parsePhpAttribute()


/**
* Creates a map for the attributes tokens that surround other tokens.
*
* @return void
*/
private function createAttributesNestingMap()
{
$map = [];
for ($i = 0; $i < $this->numTokens; $i++) {
if (isset($this->tokens[$i]['attribute_opener']) === true
&& $i === $this->tokens[$i]['attribute_opener']
) {
if (empty($map) === false) {
$this->tokens[$i]['nested_attributes'] = $map;
}

if (isset($this->tokens[$i]['attribute_closer']) === true) {
$map[$this->tokens[$i]['attribute_opener']]
= $this->tokens[$i]['attribute_closer'];
}
} else if (isset($this->tokens[$i]['attribute_closer']) === true
&& $i === $this->tokens[$i]['attribute_closer']
) {
array_pop($map);
if (empty($map) === false) {
$this->tokens[$i]['nested_attributes'] = $map;
}
} else {
if (empty($map) === false) {
$this->tokens[$i]['nested_attributes'] = $map;
}
}//end if
}//end for

}//end createAttributesNestingMap()


}//end class
34 changes: 34 additions & 0 deletions src/Tokenizers/Tokenizer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -740,6 +740,40 @@ private function createTokenMap()
$this->tokens[$i]['parenthesis_closer'] = $i;
$this->tokens[$opener]['parenthesis_closer'] = $i;
}//end if
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE) {
$openers[] = $i;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo str_repeat("\t", count($openers));
echo "=> Found attribute opener at $i".PHP_EOL;
}

$this->tokens[$i]['attribute_opener'] = $i;
$this->tokens[$i]['attribute_closer'] = null;
} else if ($this->tokens[$i]['code'] === T_ATTRIBUTE_END) {
$numOpeners = count($openers);
if ($numOpeners !== 0) {
$opener = array_pop($openers);
if (isset($this->tokens[$opener]['attribute_opener']) === true) {
$this->tokens[$opener]['attribute_closer'] = $i;

if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo str_repeat("\t", (count($openers) + 1));
echo "=> Found attribute closer at $i for $opener".PHP_EOL;
}

for ($x = ($opener + 1); $x <= $i; ++$x) {
if (isset($this->tokens[$x]['attribute_closer']) === true) {
continue;
}

$this->tokens[$x]['attribute_opener'] = $opener;
$this->tokens[$x]['attribute_closer'] = $i;
}
} else if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo str_repeat("\t", (count($openers) + 1));
echo "=> Found unowned attribute closer at $i for $opener".PHP_EOL;
}
}//end if
}//end if

/*
Expand Down
5 changes: 5 additions & 0 deletions src/Util/Tokens.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
define('T_PARAM_NAME', 'PHPCS_T_PARAM_NAME');
define('T_MATCH_ARROW', 'PHPCS_T_MATCH_ARROW');
define('T_MATCH_DEFAULT', 'PHPCS_T_MATCH_DEFAULT');
define('T_ATTRIBUTE_END', 'PHPCS_T_ATTRIBUTE_END');

// Some PHP 5.5 tokens, replicated for lower versions.
if (defined('T_FINALLY') === false) {
Expand Down Expand Up @@ -149,6 +150,10 @@
define('T_MATCH', 'PHPCS_T_MATCH');
}

if (defined('T_ATTRIBUTE') === false) {
define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
}

// Tokens used for parsing doc blocks.
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
Expand Down
16 changes: 16 additions & 0 deletions tests/Core/File/GetMemberPropertiesTest.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,3 +240,19 @@ $anon = class() {
// Intentional fatal error - duplicate types are not allowed in union types, but that's not the concern of the method.
public int |string| /*comment*/ INT $duplicateTypeInUnion;
};

$anon = class {
/* testPHP8PropertySingleAttribute */
#[PropertyWithAttribute]
public string $foo;

/* testPHP8PropertyMultipleAttributes */
#[PropertyWithAttribute(foo: 'bar'), MyAttribute]
protected ?int|float $bar;

/* testPHP8PropertyMultilineAttribute */
#[
PropertyWithAttribute(/* comment */ 'baz')
]
private mixed $baz;
};
30 changes: 30 additions & 0 deletions tests/Core/File/GetMemberPropertiesTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,36 @@ public function dataGetMemberProperties()
'nullable_type' => false,
],
],
[
'/* testPHP8PropertySingleAttribute */',
[
'scope' => 'public',
'scope_specified' => true,
'is_static' => false,
'type' => 'string',
'nullable_type' => false,
],
],
[
'/* testPHP8PropertyMultipleAttributes */',
[
'scope' => 'protected',
'scope_specified' => true,
'is_static' => false,
'type' => '?int|float',
'nullable_type' => true,
],
],
[
'/* testPHP8PropertyMultilineAttribute */',
[
'scope' => 'private',
'scope_specified' => true,
'is_static' => false,
'type' => 'mixed',
'nullable_type' => false,
],
],
];

}//end dataGetMemberProperties()
Expand Down
Loading

0 comments on commit 013943d

Please sign in to comment.