Skip to content

Commit

Permalink
PHP 8.0 | Tokenizer/PHP: tokenize the "|" for union types as T_TYPE_U…
Browse files Browse the repository at this point in the history
…NION

This adds a new block of logic to the `PHP::processAdditional()` method which changes the token code and type of `T_BITWISE_OR` `|` tokens in type declarations to `T_TYPE_UNION`.

As the `PHP::processAdditional()` method walks backwards through the token stack, the arrow function backfill will not have been done yet, so for those some special conditions have been put in place.

I've tried to limit the token walking within the new block as much as possible while still maintaining accuracy.

This includes changing all union type operators in a single type declaration in one go, instead of on each individual `T_BITWISE_OR` token, which prevents the same logic having to be executed multiple times for multi-union types like `int|float|null`.

Includes dedicated unit tests.

Ref: https://wiki.php.net/rfc/union_types_v2
  • Loading branch information
jrfnl committed Sep 20, 2020
1 parent a0145ae commit 63c2b22
Show file tree
Hide file tree
Showing 4 changed files with 384 additions and 0 deletions.
6 changes: 6 additions & 0 deletions package.xml
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
<file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
<file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
<file baseinstalldir="" name="BitwiseOrTest.inc" role="test" />
<file baseinstalldir="" name="BitwiseOrTest.php" role="test" />
<file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
<file baseinstalldir="" name="NullsafeObjectOperatorTest.php" role="test" />
<file baseinstalldir="" name="ScopeSettingWithNamespaceOperatorTest.inc" role="test" />
Expand Down Expand Up @@ -2006,6 +2008,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" name="tests/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" />
Expand Down Expand Up @@ -2071,6 +2075,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.php" name="tests/Core/Tokenizer/BitwiseOrTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" name="tests/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" />
Expand Down
171 changes: 171 additions & 0 deletions src/Tokenizers/PHP.php
Original file line number Diff line number Diff line change
Expand Up @@ -2176,6 +2176,177 @@ protected function processAdditional()
}
}

continue;
} else if ($this->tokens[$i]['code'] === T_BITWISE_OR) {
/*
Convert "|" to T_TYPE_UNION or leave as T_BITWISE_OR.
*/

$allowed = [
T_STRING => T_STRING,
T_CALLABLE => T_CALLABLE,
T_SELF => T_SELF,
T_PARENT => T_PARENT,
T_STATIC => T_STATIC,
T_FALSE => T_FALSE,
T_NULL => T_NULL,
T_NS_SEPARATOR => T_NS_SEPARATOR,
];

$suspectedType = null;
$typeTokenCount = 0;

for ($x = ($i + 1); $x < $numTokens; $x++) {
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true) {
continue;
}

if (isset($allowed[$this->tokens[$x]['code']]) === true) {
++$typeTokenCount;
continue;
}

if ($typeTokenCount > 0
&& ($this->tokens[$x]['code'] === T_BITWISE_AND
|| $this->tokens[$x]['code'] === T_ELLIPSIS)
) {
// Skip past reference and variadic indicators for parameter types.
++$x;
continue;
}

if ($this->tokens[$x]['code'] === T_VARIABLE) {
// Parameter/Property defaults can not contain variables, so this could be a type.
$suspectedType = 'property or parameter';
break;
}

if ($this->tokens[$x]['code'] === T_DOUBLE_ARROW) {
// Possible arrow function.
$suspectedType = 'return';
break;
}

if ($this->tokens[$x]['code'] === T_SEMICOLON) {
// Possible abstract method or interface method.
$suspectedType = 'return';
break;
}

if ($this->tokens[$x]['code'] === T_OPEN_CURLY_BRACKET
&& isset($this->tokens[$x]['scope_condition']) === true
&& $this->tokens[$this->tokens[$x]['scope_condition']]['code'] === T_FUNCTION
) {
$suspectedType = 'return';
}

break;
}//end for

if ($typeTokenCount === 0 || isset($suspectedType) === false) {
// Definitely not a union type, move on.
continue;
}

$typeTokenCount = 0;
$unionOperators = [$i];
$confirmed = false;

for ($x = ($i - 1); $x >= 0; $x--) {
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true) {
continue;
}

if (isset($allowed[$this->tokens[$x]['code']]) === true) {
++$typeTokenCount;
continue;
}

// Union types can't use the nullable operator, but be tolerant to parse errors.
if ($typeTokenCount > 0 && $this->tokens[$x]['code'] === T_NULLABLE) {
continue;
}

if ($this->tokens[$x]['code'] === T_BITWISE_OR) {
$unionOperators[] = $x;
continue;
}

if ($suspectedType === 'return' && $this->tokens[$x]['code'] === T_COLON) {
$confirmed = true;
break;
}

if ($suspectedType === 'property or parameter'
&& (isset(Util\Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true
|| $this->tokens[$x]['code'] === T_VAR)
) {
// This will also confirm constructor property promotion parameters, but that's fine.
$confirmed = true;
}

break;
}//end for

if ($confirmed === false
&& $suspectedType === 'property or parameter'
&& isset($this->tokens[$i]['nested_parenthesis']) === true
) {
$parens = $this->tokens[$i]['nested_parenthesis'];
$last = end($parens);

if (isset($this->tokens[$last]['parenthesis_owner']) === true
&& $this->tokens[$this->tokens[$last]['parenthesis_owner']]['code'] === T_FUNCTION
) {
$confirmed = true;
} else {
// No parenthesis owner set, this may be an arrow function which has not yet
// had additional processing done.
if (isset($this->tokens[$last]['parenthesis_opener']) === true) {
for ($x = ($this->tokens[$last]['parenthesis_opener'] - 1); $x >= 0; $x--) {
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true) {
continue;
}

break;
}

if ($this->tokens[$x]['code'] === T_FN) {
for (--$x; $x >= 0; $x--) {
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === true
|| $this->tokens[$x]['code'] === T_BITWISE_AND
) {
continue;
}

break;
}

if ($this->tokens[$x]['code'] !== T_FUNCTION) {
$confirmed = true;
}
}
}//end if
}//end if

unset($parens, $last);
}//end if

if ($confirmed === false) {
// Not a union type after all, move on.
continue;
}

foreach ($unionOperators as $x) {
$this->tokens[$x]['code'] = T_TYPE_UNION;
$this->tokens[$x]['type'] = 'T_TYPE_UNION';

if (PHP_CODESNIFFER_VERBOSITY > 1) {
$line = $this->tokens[$x]['line'];
echo "\t* token $x on line $line changed from T_BITWISE_OR to T_TYPE_UNION".PHP_EOL;
}
}

continue;
} else if ($this->tokens[$i]['code'] === T_STATIC) {
for ($x = ($i - 1); $x > 0; $x--) {
Expand Down
85 changes: 85 additions & 0 deletions tests/Core/Tokenizer/BitwiseOrTest.inc
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
<?php

/*
* Type union or bitwise or.
*/

/* testBitwiseOr1 */
$result = $value | $test /* testBitwiseOr2 */ | $another;

class TypeUnion
{
/* testTypeUnionPropertySimple */
public static Foo|Bar $obj;

/* testTypeUnionPropertyReverseModifierOrder */
static protected int|float $number /* testBitwiseOrPropertyDefaultValue */ = E_WARNING | E_NOTICE;

private
/* testTypeUnionPropertyMulti1 */
array |
/* testTypeUnionPropertyMulti2 */
Traversable | // phpcs:ignore Stnd.Cat.Sniff
false
/* testTypeUnionPropertyMulti3 */
| null $arrayOrFalse;

public function paramTypes(
/* testTypeUnionParam1 */
int|float $paramA /* testBitwiseOrParamDefaultValue */ = CONSTANT_A | CONSTANT_B,

/* testTypeUnionParam2 */
Foo|\Bar /* testTypeUnionParam3 */ |Baz &...$paramB = null,
) {
/* testBitwiseOr3 */
return (($a1 ^ $b1) |($a2 ^ $b2)) + $c;
}

/* testTypeUnionReturnType */
public function returnType() : int|false {}

/* testTypeUnionConstructorPropertyPromotion */
public function __construct( public bool|null $property) {}

/* testTypeUnionAbstractMethodReturnType1 */
abstract public function abstractMethod(): object|array /* testTypeUnionAbstractMethodReturnType2 */ |false;
}

/* testTypeUnionClosureParamIllegalNullable */
$closureWithParamType = function (?string|null $string) {};

/* testBitwiseOrClosureParamDefault */
$closureWithReturnType = function ($string = NONSENSE | FAKE)/* testTypeUnionClosureReturn */ : \Package\MyA|PackageB {};

/* testTypeUnionArrowParam */
$arrowWithParamType = fn (object|array $param, /* testBitwiseOrArrowParamDefault */ ?int $int = CONSTA | CONSTB )
/* testBitwiseOrArrowExpression */
=> $param | $int;

/* testTypeUnionArrowReturnType */
$arrowWithReturnType = fn ($param) : int|null => $param * 10;

/* testBitwiseOrInArrayKey */
$array = array(
A | B => /* testBitwiseOrInArrayValue */ B | C
);

/* testBitwiseOrInShortArrayKey */
$array = [
A | B => /* testBitwiseOrInShortArrayValue */ B | C
];

/* testBitwiseOrTryCatch */
try {
} catch ( ExceptionA | ExceptionB $e ) {
}

/* testBitwiseOrNonArrowFnFunctionCall */
$obj->fn($something | $else);

/* testTypeUnionNonArrowFunctionDeclaration */
function &fn(int|false $something) {}

/* testLiveCoding */
// Intentional parse error. This has to be the last test in the file.
return function( type|
Loading

0 comments on commit 63c2b22

Please sign in to comment.