Skip to content

Commit

Permalink
Merge branch 'feature/fix-php-7.4-numeric-separators-backfill' of htt…
Browse files Browse the repository at this point in the history
  • Loading branch information
gsherwood committed Jan 6, 2020
2 parents be86839 + bb9216e commit 1757e77
Show file tree
Hide file tree
Showing 3 changed files with 366 additions and 21 deletions.
36 changes: 26 additions & 10 deletions src/Tokenizers/PHP.php
Original file line number Diff line number Diff line change
Expand Up @@ -992,20 +992,24 @@ protected function tokenize($string)

if ($tokens[$i][0] === T_LNUMBER
|| $tokens[$i][0] === T_DNUMBER
|| ($tokens[$i][0] === T_STRING
&& $tokens[$i][1][0] === '_')
) {
$newContent .= $tokens[$i][1];
continue;
}

// Any T_DNUMBER token needs to make the
// new number a T_DNUMBER as well.
if ($tokens[$i][0] === T_DNUMBER) {
$newType = T_DNUMBER;
}
if ($tokens[$i][0] === T_STRING
&& $tokens[$i][1][0] === '_'
&& ((strpos($newContent, '0x') === 0
&& preg_match('`^((?<!\.)_[0-9A-F][0-9A-F\.]*)+$`iD', $tokens[$i][1]) === 1)
|| (strpos($newContent, '0x') !== 0
&& substr($newContent, -1) !== '.'
&& substr(strtolower($newContent), -1) !== 'e'
&& preg_match('`^(?:(?<![\.e])_[0-9][0-9e\.]*)+$`iD', $tokens[$i][1]) === 1))
) {
$newContent .= $tokens[$i][1];

// Support floats.
if ($tokens[$i][0] === T_STRING
&& substr(strtolower($tokens[$i][1]), -1) === 'e'
if (substr(strtolower($tokens[$i][1]), -1) === 'e'
&& ($tokens[($i + 1)] === '-'
|| $tokens[($i + 1)] === '+')
) {
Expand All @@ -1019,9 +1023,21 @@ protected function tokenize($string)
break;
}//end for

if ($newType === T_LNUMBER
&& ((stripos($newContent, '0x') === 0 && hexdec(str_replace('_', '', $newContent)) > PHP_INT_MAX)
|| (stripos($newContent, '0b') === 0 && bindec(str_replace('_', '', $newContent)) > PHP_INT_MAX)
|| (stripos($newContent, '0x') !== 0
&& stripos($newContent, 'e') !== false || strpos($newContent, '.') !== false)
|| (strpos($newContent, '0') === 0 && stripos($newContent, '0x') !== 0
&& stripos($newContent, '0b') !== 0 && octdec(str_replace('_', '', $newContent)) > PHP_INT_MAX)
|| (strpos($newContent, '0') !== 0 && str_replace('_', '', $newContent) > PHP_INT_MAX))
) {
$newType = T_DNUMBER;
}

$newToken = [];
$newToken['code'] = $newType;
$newToken['type'] = Util\Tokens::tokenName($token[0]);
$newToken['type'] = Util\Tokens::tokenName($newType);
$newToken['content'] = $newContent;
$finalTokens[$newStackPtr] = $newToken;

Expand Down
59 changes: 58 additions & 1 deletion tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc
Original file line number Diff line number Diff line change
@@ -1,25 +1,82 @@
<?php

/*
* Numbers with PHP 7.4 underscore separators.
*/

/* testSimpleLNumber */
$foo = 1_000_000_000;

/* testSimpleDNumber */
$foo = 107_925_284.88;
$foo = 107_925_284.88;

/* testFloat */
$foo = 6.674_083e-11;

/* testFloat2 */
$foo = 6.674_083e+11;

/* testFloat3 */
$foo = 1_2.3_4e1_23;

/* testHex */
$foo = 0xCAFE_F00D;

/* testHexMultiple */
$foo = 0x42_72_6F_77_6E;

/* testHexInt */
$foo = 0x42_72_6F;

/* testBinary */
$foo = 0b0101_1111;

/* testOctal */
$foo = 0137_041;

/* testIntMoreThanMax */
$foo = 10_223_372_036_854_775_807;

/*
* Invalid use of numeric separators. These should not be touched by the backfill.
*/

/* testInvalid1 */
$a = 100_; // trailing

/* testInvalid2 */
$a = 1__1; // next to underscore

/* testInvalid3 */
$a = 1_.0; // next to decimal point

/* testInvalid4 */
$a = 1._0; // next to decimal point

/* testInvalid5 */
$a = 0x_123; // next to x

/* testInvalid6 */
$a = 0b_101; // next to b

/* testInvalid7 */
$a = 1_e2; // next to e

/* testInvalid8 */
$a = 1e_2; // next to e

/* testInvalid9 */
$testValue = 107_925_284 .88;

/* testInvalid10 */
$testValue = 107_925_284/*comment*/.88;

/*
* Ensure that legitimate calculations are not touched by the backfill.
*/

/* testCalc1 */
$a = 667_083 - 11; // Calculation.

/* test Calc2 */
$a = 6.674_08e3 + 11; // Calculation.
Loading

0 comments on commit 1757e77

Please sign in to comment.