Tokenizer / numeric separators backfill: fix parse errors being handl…

…ed by the backfill The prevents the backfill from kicking in when confronted with invalid use of the numeric literal separator which would be a parse error in PHP 7.4 anyway. This fixes the remaining unit test failures.
squizlabs · Dec 20, 2019 · bb9216e · bb9216e
1 parent 58b2e59
commit bb9216e
Showing 1 changed file with 14 additions and 4 deletions.
diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
@@ -992,14 +992,24 @@ protected function tokenize($string)
 
                     if ($tokens[$i][0] === T_LNUMBER
                         || $tokens[$i][0] === T_DNUMBER
-                        || ($tokens[$i][0] === T_STRING
-                        && $tokens[$i][1][0] === '_')
+                    ) {
+                        $newContent .= $tokens[$i][1];
+                        continue;
+                    }
+
+                    if ($tokens[$i][0] === T_STRING
+                        && $tokens[$i][1][0] === '_'
+                        && ((strpos($newContent, '0x') === 0
+                        && preg_match('`^((?<!\.)_[0-9A-F][0-9A-F\.]*)+$`iD', $tokens[$i][1]) === 1)
+                        || (strpos($newContent, '0x') !== 0
+                        && substr($newContent, -1) !== '.'
+                        && substr(strtolower($newContent), -1) !== 'e'
+                        && preg_match('`^(?:(?<![\.e])_[0-9][0-9e\.]*)+$`iD', $tokens[$i][1]) === 1))
                     ) {
                         $newContent .= $tokens[$i][1];
 
                         // Support floats.
-                        if ($tokens[$i][0] === T_STRING
-                            && substr(strtolower($tokens[$i][1]), -1) === 'e'
+                        if (substr(strtolower($tokens[$i][1]), -1) === 'e'
                             && ($tokens[($i + 1)] === '-'
                             || $tokens[($i + 1)] === '+')
                         ) {