Skip to content

Commit

Permalink
Merge branch 'feature/3041-backport-old-identifier-tokenization-php-8…
Browse files Browse the repository at this point in the history
  • Loading branch information
gsherwood committed Sep 20, 2020
2 parents 4201fd8 + ba63323 commit 6525028
Show file tree
Hide file tree
Showing 5 changed files with 1,562 additions and 9 deletions.
6 changes: 6 additions & 0 deletions package.xml
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<file baseinstalldir="" name="StableCommentWhitespaceTest.php" role="test" />
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.inc" role="test" />
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.php" role="test" />
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.inc" role="test" />
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.php" role="test" />
</dir>
<file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
<file baseinstalldir="" name="AllTests.php" role="test" />
Expand Down Expand Up @@ -2004,6 +2006,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
</filelist>
Expand Down Expand Up @@ -2065,6 +2069,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
<ignore name="bin/phpcs.bat" />
Expand Down
109 changes: 100 additions & 9 deletions src/Tokenizers/PHP.php
Original file line number Diff line number Diff line change
Expand Up @@ -815,6 +815,81 @@ protected function tokenize($string)
continue;
}//end if

/*
As of PHP 8.0 fully qualified, partially qualified and namespace relative
identifier names are tokenized differently.
This "undoes" the new tokenization so the tokenization will be the same in
in PHP 5, 7 and 8.
*/

if (PHP_VERSION_ID >= 80000
&& $tokenIsArray === true
&& ($token[0] === T_NAME_QUALIFIED
|| $token[0] === T_NAME_FULLY_QUALIFIED
|| $token[0] === T_NAME_RELATIVE)
) {
$name = $token[1];

if ($token[0] === T_NAME_FULLY_QUALIFIED) {
$newToken = [];
$newToken['code'] = T_NS_SEPARATOR;
$newToken['type'] = 'T_NS_SEPARATOR';
$newToken['content'] = '\\';
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

$name = ltrim($name, '\\');
}

if ($token[0] === T_NAME_RELATIVE) {
$newToken = [];
$newToken['code'] = T_NAMESPACE;
$newToken['type'] = 'T_NAMESPACE';
$newToken['content'] = substr($name, 0, 9);
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

$newToken = [];
$newToken['code'] = T_NS_SEPARATOR;
$newToken['type'] = 'T_NS_SEPARATOR';
$newToken['content'] = '\\';
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

$name = substr($name, 10);
}

$parts = explode('\\', $name);
$partCount = count($parts);
$lastPart = ($partCount - 1);

foreach ($parts as $i => $part) {
$newToken = [];
$newToken['code'] = T_STRING;
$newToken['type'] = 'T_STRING';
$newToken['content'] = $part;
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;

if ($i !== $lastPart) {
$newToken = [];
$newToken['code'] = T_NS_SEPARATOR;
$newToken['type'] = 'T_NS_SEPARATOR';
$newToken['content'] = '\\';
$finalTokens[$newStackPtr] = $newToken;
++$newStackPtr;
}
}

if (PHP_CODESNIFFER_VERBOSITY > 1) {
$type = Util\Tokens::tokenName($token[0]);
$content = Util\Common::prepareForOutput($token[1]);
echo "\t\t* token $stackPtr split into individual tokens; was: $type => $content".PHP_EOL;
}

continue;
}//end if

/*
Before PHP 7.0, the "yield from" was tokenized as
T_YIELD, T_WHITESPACE and T_STRING. So look for
Expand Down Expand Up @@ -1131,7 +1206,7 @@ protected function tokenize($string)
* Check if the next non-empty token is one of the tokens which can be used
* in type declarations. If not, it's definitely a ternary.
* At this point, the only token types which need to be taken into consideration
* as potential type declarations are T_STRING, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
* as potential type declarations are identifier names, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
*/

$lastRelevantNonEmpty = null;
Expand All @@ -1148,6 +1223,9 @@ protected function tokenize($string)
}

if ($tokenType === T_STRING
|| $tokenType === T_NAME_FULLY_QUALIFIED
|| $tokenType === T_NAME_RELATIVE
|| $tokenType === T_NAME_QUALIFIED
|| $tokenType === T_ARRAY
|| $tokenType === T_NS_SEPARATOR
) {
Expand All @@ -1159,7 +1237,10 @@ protected function tokenize($string)
&& isset($lastRelevantNonEmpty) === false)
|| ($lastRelevantNonEmpty === T_ARRAY
&& $tokenType === '(')
|| ($lastRelevantNonEmpty === T_STRING
|| (($lastRelevantNonEmpty === T_STRING
|| $lastRelevantNonEmpty === T_NAME_FULLY_QUALIFIED
|| $lastRelevantNonEmpty === T_NAME_RELATIVE
|| $lastRelevantNonEmpty === T_NAME_QUALIFIED)
&& ($tokenType === T_DOUBLE_COLON
|| $tokenType === '('
|| $tokenType === ':'))
Expand Down Expand Up @@ -1304,6 +1385,10 @@ protected function tokenize($string)
tokenized as T_STRING even if it appears to be a different token,
such as when writing code like: function default(): foo
so go forward and change the token type before it is processed.
Note: this should not be done for `function Level\Name` within a
group use statement for the PHP 8 identifier name tokens as it
would interfere with the re-tokenization of those.
*/

if ($tokenIsArray === true
Expand All @@ -1321,7 +1406,10 @@ protected function tokenize($string)
}
}

if ($x < $numTokens && is_array($tokens[$x]) === true) {
if ($x < $numTokens
&& is_array($tokens[$x]) === true
&& $tokens[$x][0] !== T_NAME_QUALIFIED
) {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$oldType = Util\Tokens::tokenName($tokens[$x][0]);
echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL;
Expand Down Expand Up @@ -1377,12 +1465,15 @@ function return types. We want to keep the parenthesis map clean,
&& $tokens[$x] === ':'
) {
$allowed = [
T_STRING => T_STRING,
T_ARRAY => T_ARRAY,
T_CALLABLE => T_CALLABLE,
T_SELF => T_SELF,
T_PARENT => T_PARENT,
T_NS_SEPARATOR => T_NS_SEPARATOR,
T_STRING => T_STRING,
T_NAME_FULLY_QUALIFIED => T_NAME_FULLY_QUALIFIED,
T_NAME_RELATIVE => T_NAME_RELATIVE,
T_NAME_QUALIFIED => T_NAME_QUALIFIED,
T_ARRAY => T_ARRAY,
T_CALLABLE => T_CALLABLE,
T_SELF => T_SELF,
T_PARENT => T_PARENT,
T_NS_SEPARATOR => T_NS_SEPARATOR,
];

$allowed += Util\Tokens::$emptyTokens;
Expand Down
12 changes: 12 additions & 0 deletions src/Util/Tokens.php
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,18 @@
define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
}

if (defined('T_NAME_QUALIFIED') === false) {
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
}

if (defined('T_NAME_FULLY_QUALIFIED') === false) {
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
}

if (defined('T_NAME_RELATIVE') === false) {
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
}

// Tokens used for parsing doc blocks.
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
Expand Down
147 changes: 147 additions & 0 deletions tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
<?php

/* testNamespaceDeclaration */
namespace Package;

/* testNamespaceDeclarationWithLevels */
namespace Vendor\SubLevel\Domain;

/* testUseStatement */
use ClassName;

/* testUseStatementWithLevels */
use Vendor\Level\Domain;

/* testFunctionUseStatement */
use function function_name;

/* testFunctionUseStatementWithLevels */
use function Vendor\Level\function_in_ns;

/* testConstantUseStatement */
use const CONSTANT_NAME;

/* testConstantUseStatementWithLevels */
use const Vendor\Level\OTHER_CONSTANT;

/* testMultiUseUnqualified */
use UnqualifiedClassName,
/* testMultiUsePartiallyQualified */
Sublevel\PartiallyClassName;

/* testGroupUseStatement */
use Vendor\Level\{
AnotherDomain,
function function_grouped,
const CONSTANT_GROUPED,
Sub\YetAnotherDomain,
function SubLevelA\function_grouped_too,
const SubLevelB\CONSTANT_GROUPED_TOO,
};

/* testClassName */
class MyClass
/* testExtendedFQN */
extends \Vendor\Level\FQN
/* testImplementsRelative */
implements namespace\Name,
/* testImplementsFQN */
\Fully\Qualified,
/* testImplementsUnqualified */
Unqualified,
/* testImplementsPartiallyQualified */
Sub\Level\Name
{
/* testFunctionName */
public function function_name(
/* testTypeDeclarationRelative */
?namespace\Name|object $paramA,

/* testTypeDeclarationFQN */
\Fully\Qualified\Name $paramB,

/* testTypeDeclarationUnqualified */
Unqualified|false $paramC,

/* testTypeDeclarationPartiallyQualified */
?Sublevel\Name $paramD,

/* testReturnTypeFQN */
) : ?\Name {

try {
/* testFunctionCallRelative */
echo NameSpace\function_name();

/* testFunctionCallFQN */
echo \Vendor\Package\function_name();

/* testFunctionCallUnqualified */
echo function_name();

/* testFunctionPartiallyQualified */
echo Level\function_name();

/* testCatchRelative */
} catch (namespace\SubLevel\Exception $e) {

/* testCatchFQN */
} catch (\Exception $e) {

/* testCatchUnqualified */
} catch (Exception $e) {

/* testCatchPartiallyQualified */
} catch (Level\Exception $e) {
}

/* testNewRelative */
$obj = new namespace\ClassName();

/* testNewFQN */
$obj = new \Vendor\ClassName();

/* testNewUnqualified */
$obj = new ClassName;

/* testNewPartiallyQualified */
$obj = new Level\ClassName;

/* testDoubleColonRelative */
$value = namespace\ClassName::property;

/* testDoubleColonFQN */
$value = \ClassName::static_function();

/* testDoubleColonUnqualified */
$value = ClassName::CONSTANT_NAME;

/* testDoubleColonPartiallyQualified */
$value = Level\ClassName::CONSTANT_NAME['key'];

/* testInstanceOfRelative */
$is = $obj instanceof namespace\ClassName;

/* testInstanceOfFQN */
if ($obj instanceof \Full\ClassName) {}

/* testInstanceOfUnqualified */
if ($a === $b && $obj instanceof ClassName && true) {}

/* testInstanceOfPartiallyQualified */
$is = $obj instanceof Partially\ClassName;
}
}

/* testInvalidInPHP8Whitespace */
namespace \ Sublevel
\ function_name();

/* testInvalidInPHP8Comments */
$value = \Fully
// phpcs:ignore Stnd.Cat.Sniff -- for reasons
\Qualified
/* comment */
\Name
// comment
:: function_name();
Loading

0 comments on commit 6525028

Please sign in to comment.