New 'completions' command, and various tab-completion fixes and improvements

src/Command/CompletionsCommand.php

@@ -0,0 +1,62 @@
+<?php
+
+/*
+ * This file is part of Psy Shell.
+ *
+ * (c) 2012-2020 Justin Hileman
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Psy\Command;
+
+use Psy\Input\CodeArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Dump an array of possible completions for the given input.
+ */
+class CompletionsCommand extends Command
+{
+    /**
+     * {@inheritdoc}
+     */
+    protected function configure()
+    {
+        $this
+            ->setName('completions')
+            ->setDefinition([
+                new CodeArgument('target', CodeArgument::OPTIONAL, 'PHP code to complete.'),
+            ])
+            ->setDescription('List possible code completions for the input.')
+            ->setHelp(
+                <<<'HELP'
+This command enables PsySH wrappers to obtain completions for the current
+input, for the purpose of implementing their own completion UI.
+HELP
+            );
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $target = $input->getArgument('target');
+        if (!isset($target)) {
+            $target = '';
+        }
+
+        // n.b. All of the relevant parts of \Psy\Shell are protected
+        // or private, so getTabCompletions() itself is a Shell method.
+        $completions = $this->getApplication()->getTabCompletions($target);
+
+        // Ouput the completion candidates as newline-separated text.
+        $str = \implode("\n", \array_filter($completions))."\n";
+        $output->write($str, false, OutputInterface::OUTPUT_RAW);
+
+        return 0;
+    }
+}

src/Input/CodeArgument.php

@@ -26,6 +26,11 @@
  *     parse function() { return "wheee\n"; }
  *
  * ... without having to put the code in a quoted string and escape everything.
+ *
+ * Certain trailing whitespace characters are exceptions.  Trailing Spaces and
+ * tabs will be included in the argument value, but trailing newlines, carriage
+ * returns, vertical tabs, and nulls are trimmed from the command before the
+ * arguments are established.
  */
 class CodeArgument extends InputArgument
 {

src/Shell.php

@@ -22,6 +22,7 @@
 use Psy\Formatter\TraceFormatter;
 use Psy\Input\ShellInput;
 use Psy\Input\SilentInput;
+use Psy\TabCompletion\AutoCompleter;
 use Psy\TabCompletion\Matcher;
 use Psy\VarDumper\PresenterAware;
 use Symfony\Component\Console\Application;
@@ -237,6 +238,24 @@ protected function getTabCompletionMatchers()
         @\trigger_error('getTabCompletionMatchers is no longer used', \E_USER_DEPRECATED);
     }
 
+    /**
+     * Get completion matches.
+     *
+     * @return array An array of completion matches for $input
+     */
+    public function getTabCompletions(string $input)
+    {
+        $ac = $this->autoCompleter;
+        $word = '';
+        $regexp = AutoCompleter::WORD_REGEXP;
+        $matches = [];
+        if (\preg_match($regexp, $input, $matches) === 1) {
+            $word = $matches[0];
+        }
+
+        return $ac->processCallback($word, null, ['line_buffer' => $input]);
+    }
+
     /**
      * Gets the default command loop listeners.
      *
@@ -909,7 +928,7 @@ protected function runCommand($input)
             throw new \InvalidArgumentException('Command not found: '.$input);
         }
 
-        $input = new ShellInput(\str_replace('\\', '\\\\', \rtrim($input, " \t\n\r\0\x0B;")));
+        $input = new ShellInput(\str_replace('\\', '\\\\', \rtrim($input, "\n\r\0\x0B;")));
 
         if ($input->hasParameterOption(['--help', '-h'])) {
             $helpCommand = $this->get('help');

src/TabCompletion/AutoCompleter.php

@@ -23,6 +23,84 @@ class AutoCompleter
     /** @var Matcher\AbstractMatcher[] */
     protected $matchers;
 
+    /**
+     * The set of characters which separate completeable 'words', and
+     * therefore determines the precise $input word for which completion
+     * candidates should be generated (noting that each candidate must
+     * begin with the original $input text).
+     *
+     * PHP's readline support does not provide any control over the
+     * characters which constitute a word break for completion purposes,
+     * which means that we are restricted to the default -- being the
+     * value of GNU Readline's rl_basic_word_break_characters variable:
+     *
+     *   The basic list of characters that signal a break between words
+     *   for the completer routine. The default value of this variable
+     *   is the characters which break words for completion in Bash:
+     *   " \t\n\"\\’‘@$><=;|&{(".
+     *
+     * This limitation has several ramifications for PHP completion:
+     *
+     *  1. The namespace separator '\' introduces a word break, and so
+     *     class name completion is on a per-namespace-component basis.
+     *     When completing a namespaced class, the (incomplete) $input
+     *     parameter (and hence the completion candidates we return) will
+     *     not include the separator or preceding namespace components.
+     *
+     *  2. The double-colon (nekudotayim) operator '::' does NOT introduce
+     *     a word break (as ':' is not a word break character), and so the
+     *     $input parameter will include the preceding 'ClassName::' text
+     *     (typically back to, but not including, a space character or
+     *     namespace-separator '\').  Completion candidates for class
+     *     attributes and methods must therefore include this same prefix.
+     *
+     *  3. The object operator '->' introduces a word break (as '>' is a
+     *     word break character), so when completing an object attribute
+     *     or method, $input will contain only the text following the
+     *     operator, and therefore (unlike '::') the completion candidates
+     *     we return must NOT include the preceding object and operator.
+     *
+     *  4. '$' is a word break character, and so completion for variable
+     *     names does not include the leading '$'.  The $input parameter
+     *     contains only the text following the '$' and therefore the
+     *     candidates we return must do likewise...
+     *
+     *  5. ...Except when we are returning ALL variables (amongst other
+     *     things) as candidates for completing the empty string '', in
+     *     which case we DO need to include the '$' character in each of
+     *     our candidates, because it was not already present in the text.
+     *     (Note that $input will be '' when we are completing either ''
+     *     or '$', so we need to distinguish between those two cases.)
+     *
+     *  6. Only a sub-set of other PHP operators constitute (or end with)
+     *     word breaks, and so inconsistent behaviour can be expected if
+     *     operators are used without surrounding whitespace to ensure a
+     *     word break has occurred.
+     *
+     *     Operators which DO break words include: '>' '<' '<<' '>>' '<>'
+     *     '=' '==' '===' '!=' '!==' '>=' '<=' '<=>' '->' '|' '||' '&' '&&'
+     *     '+=' '-=' '*=' '/=' '.=' '%=' '&=' '|=' '^=' '>>=' '<<=' '??='
+     *
+     *     Operators which do NOT break words include: '!' '+' '-' '*' '/'
+     *     '++' '--' '**' '%' '.' '~' '^' '??' '? :' '::'
+     *
+     *     E.g.: although "foo()+bar()" is valid PHP, we would be unable
+     *     to use completion to obtain the function name "bar" in that
+     *     situation, as the $input string would actually begin with ")+"
+     *     and the Matcher in question would not be returning candidates
+     *     with that prefix.
+     *
+     * @see self::processCallback()
+     * @see \Psy\Shell::getTabCompletions()
+     */
+    const WORD_BREAK_CHARS = " \t\n\"\\’‘@$><=;|&{(";
+
+    /**
+     * A regular expression based on WORD_BREAK_CHARS which will match the
+     * completable word at the end of the string.
+     */
+    const WORD_REGEXP = "/[^ \t\n\"\\\\’‘@$><=;|&{(]*$/";
+
     /**
      * Register a tab completion Matcher.
      *
@@ -42,7 +120,13 @@ public function activate()
     }
 
     /**
-     * Handle readline completion.
+     * Handle readline completion for the $input parameter (word).
+     *
+     * @see WORD_BREAK_CHARS
+     *
+     * @TODO: Post-process the completion candidates returned by each
+     * Matcher to ensure that they use the common prefix established by
+     * the $input parameter.
      *
      * @param string $input Readline current word
      * @param int    $index Current word index
@@ -64,11 +148,44 @@ public function processCallback($input, $index, $info = [])
 
         $tokens = \token_get_all('<?php '.$line);
 
-        // remove whitespaces
+        // Remove whitespace tokens, excluding the current token.
+        $token = \array_pop($tokens);
         $tokens = \array_filter($tokens, function ($token) {
             return !AbstractMatcher::tokenIs($token, AbstractMatcher::T_WHITESPACE);
         });
 
+        // If we're effectively completing an empty string, append a
+        // token to represent this.  An empty string isn't actually a
+        // valid token, so we use the value that is simplest to test.
+        switch (true) {
+            case AbstractMatcher::tokenIs($token, AbstractMatcher::T_WHITESPACE):
+                $tokens[] = '';
+                break;
+            case AbstractMatcher::tokenIs($token, AbstractMatcher::T_VARIABLE):
+            case $token === '$':
+                // We allow a special case for '$', which for completion
+                // purposes we will treat the same way as T_VARIABLE.
+                $tokens[] = $token;
+                break;
+            case !AbstractMatcher::tokenIsValidIdentifier($token):
+                // This also covers/includes the cases !is_array($token)
+                // (which is a super-set of AbstractMatcher::isOperator()),
+                // and also tokenIs($token, AbstractMatcher::T_OPEN_TAG).
+                // Therefore it will never be the case that one of those
+                // things is the final token in the array; they can only
+                // ever be the *previous* token.  Moreover, the final token
+                // is always a valid completion prefix or else the empty
+                // string.  This simplifies the set of cases that Matchers
+                // need to cater for.
+                $tokens[] = $token;
+                $tokens[] = '';
+                break;
+            default:
+                // We're completing a valid identifier.
+                $tokens[] = $token;
+                break;
+        }
+
         $matches = [];
         foreach ($this->matchers as $matcher) {
             if ($matcher->hasMatched($tokens)) {

src/TabCompletion/Matcher/AbstractContextAwareMatcher.php

@@ -56,10 +56,25 @@ protected function getVariable($var)
     /**
      * Get all variables in the current Context.
      *
+     * The '$' prefix for each variable name is not included by default.
+     *
+     * @param bool $dollarPrefix Whether to prefix '$' to each name
+     *
      * @return array
      */
-    protected function getVariables()
+    protected function getVariables($dollarPrefix = false)
     {
-        return $this->context->getAll();
+        $variables = $this->context->getAll();
+        if (!$dollarPrefix) {
+            return $variables;
+        } else {
+            // Add '$' prefix to each name.
+            $newvars = [];
+            foreach ($variables as $name => $value) {
+                $newvars['$'.$name] = $value;
+            }
+
+            return $newvars;
+        }
     }
 }