tokenizer: add support for php8 attributes

squizlabs · Mar 19, 2021 · e0152a7 · e0152a7
1 parent 18c27ed
commit e0152a7
Show file tree

Hide file tree

Showing 9 changed files with 593 additions and 0 deletions.
diff --git a/package.xml b/package.xml
@@ -226,6 +226,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
      <dir name="Tokenizer">
       <file baseinstalldir="" name="AnonClassParenthesisOwnerTest.inc" role="test" />
       <file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
+      <file baseinstalldir="" name="AttributesTest.inc" role="test" />
+      <file baseinstalldir="" name="AttributesTest.php" role="test" />
       <file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
       <file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
       <file baseinstalldir="" name="BackfillMatchTokenTest.inc" role="test" />
@@ -2138,6 +2140,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
    <install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />
@@ -2222,6 +2226,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
    <install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillMatchTokenTest.php" name="tests/Core/Tokenizer/BackfillMatchTokenTest.php" />

diff --git a/src/Files/File.php b/src/Files/File.php
@@ -1813,6 +1813,7 @@ public function getMemberProperties($stackPtr)
                 T_SEMICOLON,
                 T_OPEN_CURLY_BRACKET,
                 T_CLOSE_CURLY_BRACKET,
+                T_ATTRIBUTE_END,
             ],
             ($stackPtr - 1)
         );

diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
@@ -903,6 +903,50 @@ protected function tokenize($string)
                 continue;
             }//end if
 
+            /*
+                PHP 8.0 Attributes
+            */
+
+            if (PHP_VERSION_ID < 80000
+                && $token[0] === T_COMMENT
+                && strpos($token[1], '#[') === 0
+            ) {
+                $subTokens = $this->parsePhpAttribute($tokens, $stackPtr);
+                if ($subTokens !== null) {
+                    array_splice($tokens, $stackPtr, 1, $subTokens);
+                    $numTokens = count($tokens);
+
+                    $tokenIsArray = true;
+                    $token        = $tokens[$stackPtr];
+                } else {
+                    $token[0] = T_ATTRIBUTE;
+                }
+            }
+
+            if ($tokenIsArray === true
+                && $token[0] === T_ATTRIBUTE
+            ) {
+                // Go looking for the close bracket.
+                $bracketCloser = $this->findCloser($tokens, ($stackPtr + 1), '[', ']');
+
+                $newToken            = [];
+                $newToken['code']    = T_ATTRIBUTE;
+                $newToken['type']    = 'T_ATTRIBUTE';
+                $newToken['content'] = '#[';
+                $finalTokens[$newStackPtr] = $newToken;
+
+                $tokens[$bracketCloser]    = [];
+                $tokens[$bracketCloser][0] = T_ATTRIBUTE_END;
+                $tokens[$bracketCloser][1] = ']';
+
+                if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                    echo "\t\t* token $bracketCloser changed from T_CLOSE_SQUARE_BRACKET to T_ATTRIBUTE_END".PHP_EOL;
+                }
+
+                $newStackPtr++;
+                continue;
+            }//end if
+
             /*
                 Tokenize the parameter labels for PHP 8.0 named parameters as a special T_PARAM_NAME
                 token and ensure that the colon after it is always T_COLON.
@@ -1845,6 +1889,7 @@ function return types. We want to keep the parenthesis map clean,
                         T_CLASS                    => true,
                         T_EXTENDS                  => true,
                         T_IMPLEMENTS               => true,
+                        T_ATTRIBUTE                => true,
                         T_NEW                      => true,
                         T_CONST                    => true,
                         T_NS_SEPARATOR             => true,
@@ -3077,4 +3122,72 @@ public static function resolveSimpleToken($token)
     }//end resolveSimpleToken()
 
 
+    /**
+     * Finds a "closer" token (closing parenthesis or square bracket for example)
+     * Handle parenthesis balancing while searching for closing token
+     *
+     * @param array  $tokens     The list of tokens to iterate searching the closing token (as returned by token_get_all)
+     * @param int    $start      The starting position
+     * @param string $openerChar The opening character
+     * @param string $closerChar The closing character
+     *
+     * @return int|null The position of the closing token, if found. NULL otherwise.
+     */
+    private function findCloser(array &$tokens, $start, $openerChar, $closerChar)
+    {
+        $numTokens = count($tokens);
+        $stack     = [0];
+        $closer    = null;
+        for ($x = $start; $x < $numTokens; $x++) {
+            if ($tokens[$x] === $openerChar) {
+                $stack[] = $x;
+            } else if ($tokens[$x] === $closerChar) {
+                array_pop($stack);
+                if (empty($stack) === true) {
+                    $closer = $x;
+                    break;
+                }
+            }
+        }
+
+        return $closer;
+
+    }//end findCloser()
+
+
+    /**
+     * PHP 8 attributes parser for PHP < 8
+     * Handles single-line and multiline attributes.
+     *
+     * @param array $tokens   The original array of tokens (as returned by token_get_all)
+     * @param int   $stackPtr The current position in token array
+     *
+     * @return array|null The array of parsed attribute tokens
+     */
+    private function parsePhpAttribute(array &$tokens, $stackPtr)
+    {
+
+        $token = $tokens[$stackPtr];
+
+        $commentBody = substr($token[1], 2);
+        $subTokens   = @token_get_all('<?php '.$commentBody);
+        array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
+
+        // Go looking for the close bracket.
+        $bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
+        if ($bracketCloser === null) {
+            $bracketCloser = $this->findCloser($tokens, $stackPtr, '[', ']');
+            if ($bracketCloser === null) {
+                return null;
+            }
+
+            array_splice($subTokens, count($subTokens), 0, array_slice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr)));
+            array_splice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr));
+        }
+
+        return $subTokens;
+
+    }//end parsePhpAttribute()
+
+
 }//end class
diff --git a/src/Tokenizers/Tokenizer.php b/src/Tokenizers/Tokenizer.php
@@ -740,6 +740,23 @@ private function createTokenMap()
                     $this->tokens[$i]['parenthesis_closer']      = $i;
                     $this->tokens[$opener]['parenthesis_closer'] = $i;
                 }//end if
+            } else if ($this->tokens[$i]['code'] === T_ATTRIBUTE) {
+                $found     = null;
+                $numTokens = count($this->tokens);
+                for ($x = ($i + 1); $x < $numTokens; $x++) {
+                    if ($this->tokens[$x]['code'] === T_ATTRIBUTE_END) {
+                        $found = $x;
+                        break;
+                    }
+                }
+
+                $this->tokens[$i]['attribute_opener'] = $i;
+                $this->tokens[$i]['attribute_closer'] = $found;
+
+                if ($found !== null) {
+                    $this->tokens[$found]['attribute_opener'] = $i;
+                    $this->tokens[$found]['attribute_closer'] = $found;
+                }
             }//end if
 
             /*

diff --git a/src/Util/Tokens.php b/src/Util/Tokens.php
@@ -79,6 +79,7 @@
 define('T_PARAM_NAME', 'PHPCS_T_PARAM_NAME');
 define('T_MATCH_ARROW', 'PHPCS_T_MATCH_ARROW');
 define('T_MATCH_DEFAULT', 'PHPCS_T_MATCH_DEFAULT');
+define('T_ATTRIBUTE_END', 'PHPCS_T_ATTRIBUTE_END');
 
 // Some PHP 5.5 tokens, replicated for lower versions.
 if (defined('T_FINALLY') === false) {
@@ -149,6 +150,10 @@
     define('T_MATCH', 'PHPCS_T_MATCH');
 }
 
+if (defined('T_ATTRIBUTE') === false) {
+    define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
+}
+
 // Tokens used for parsing doc blocks.
 define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
 define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

diff --git a/tests/Core/File/GetMemberPropertiesTest.inc b/tests/Core/File/GetMemberPropertiesTest.inc
@@ -240,3 +240,19 @@ $anon = class() {
     // Intentional fatal error - duplicate types are not allowed in union types, but that's not the concern of the method.
     public int |string| /*comment*/ INT $duplicateTypeInUnion;
 };
+
+$anon = class {
+    /* testPHP8PropertySingleAttribute */
+    #[PropertyWithAttribute]
+    public string $foo;
+
+    /* testPHP8PropertyMultipleAttributes */
+    #[PropertyWithAttribute(foo: 'bar'), MyAttribute]
+    protected ?int|float $bar;
+
+    /* testPHP8PropertyMultilineAttribute */
+    #[
+        PropertyWithAttribute(/* comment */ 'baz')
+    ]
+    private mixed $baz;
+};
diff --git a/tests/Core/File/GetMemberPropertiesTest.php b/tests/Core/File/GetMemberPropertiesTest.php
@@ -610,6 +610,36 @@ public function dataGetMemberProperties()
                     'nullable_type'   => false,
                 ],
             ],
+            [
+                '/* testPHP8PropertySingleAttribute */',
+                [
+                    'scope'           => 'public',
+                    'scope_specified' => true,
+                    'is_static'       => false,
+                    'type'            => 'string',
+                    'nullable_type'   => false,
+                ],
+            ],
+            [
+                '/* testPHP8PropertyMultipleAttributes */',
+                [
+                    'scope'           => 'protected',
+                    'scope_specified' => true,
+                    'is_static'       => false,
+                    'type'            => '?int|float',
+                    'nullable_type'   => true,
+                ],
+            ],
+            [
+                '/* testPHP8PropertyMultilineAttribute */',
+                [
+                    'scope'           => 'private',
+                    'scope_specified' => true,
+                    'is_static'       => false,
+                    'type'            => 'mixed',
+                    'nullable_type'   => false,
+                ],
+            ],
         ];
 
     }//end dataGetMemberProperties()

diff --git a/tests/Core/Tokenizer/AttributesTest.inc b/tests/Core/Tokenizer/AttributesTest.inc
@@ -0,0 +1,69 @@
+<?php
+
+/* testAttribute */
+#[Attribute]
+class CustomAttribute {}
+
+/* testAttributeWithParams */
+#[Attribute(Attribute::TARGET_CLASS)]
+class SecondCustomAttribute {}
+
+/* testAttributeWithNamedParam */
+#[Attribute(flags: Attribute::TARGET_ALL)]
+class AttributeWithParams {
+    public function __construct($foo, array $bar) {}
+}
+
+/* testAttributeOnFunction */
+#[CustomAttribute]
+function attribute_on_function_test() {}
+
+/* testAttributeOnFunctionWithParams */
+#[AttributeWithParams('foo', bar: ['bar' => 'foobar'])]
+function attribute_with_params_on_function_test() {}
+
+/* testTwoAttributeOnTheSameLine */
+#[CustomAttribute] #[AttributeWithParams('foo')]
+function two_attribute_on_same_line_test() {}
+
+/* testAttributeAndCommentOnTheSameLine */
+#[CustomAttribute] // This is a comment
+function attribute_and_line_comment_on_same_line_test() {}
+
+/* testAttributeGrouping */
+#[CustomAttribute, AttributeWithParams('foo'), AttributeWithParams('foo', bar: ['bar' => 'foobar'])]
+function attribute_grouping_test() {}
+
+/* testAttributeMultiline */
+#[
+    CustomAttribute,
+    AttributeWithParams('foo'),
+    AttributeWithParams('foo', bar: ['bar' => 'foobar'])
+]
+function attribute_multiline_test() {}
+
+/* testAttributeMultilineWithComment */
+#[
+    CustomAttribute,                // comment
+    AttributeWithParams(/* another comment */ 'foo'),
+    AttributeWithParams('foo', bar: ['bar' => 'foobar'])
+]
+function attribute_multiline_with_comment_test() {}
+
+/* testSingleAttributeOnParameter */
+function single_attribute_on_parameter_test(#[ParamAttribute] int $param) {}
+
+/* testMultipleAttributesOnParameter */
+function multiple_attributes_on_parameter_test(#[ParamAttribute, AttributeWithParams(/* another comment */ 'foo')] int $param) {}
+
+/* testMultilineAttributesOnParameter */
+function multiline_attributes_on_parameter_test(#[
+    AttributeWithParams(
+        'foo'
+    )
+                                                ] int $param) {}
+
+/* testInvalidAttribute */
+#[ThisIsNotAnAttribute
+function invalid_attribute_test() {}
+