From fb69f86eafedbcf0e2ab830c9f49bad0d3300d37 Mon Sep 17 00:00:00 2001 From: "Eloy Lafuente (stronk7)" Date: Sun, 24 Apr 2022 18:46:06 +0200 Subject: [PATCH] Use pcov => xdebug => phpdbg for phpunit code coverage Right now we are only using phpdbg for phpunit code coverage. And it's causing some problems with latest (8.0, 8.1) php versions. Also, since phpunit 8, its code coverage tool supports pcov, that is the lighter and better alternative. And then, xdebug running in coverage mode has become way better in recent versions. So, with this patch applied, moodle-plugin-ci will look for the "best" code coverage alternative (pcov => xdebug => phpdbg) in every run. This guarantees that existing automations, not having pcov or xdebug available will continue fall-backing to phpdbg. But if any of the 2 extensions are available, we'll be using them. No matter of the above (fully automatic), it's also possible to configure the desired code coverage tool with 3 new options, in case it's needed to force any: - coverage-pcov - coverage-xdebug - coverage-phpdbg Added support to the '--verbose' option to be passed to phpunit (useful to debug options or see details about skipped tests). Includes modifications to both GHA docs and template to show the new options. And, also, changes to own CI tests to have both pcov and xdebug used all the time. --- .github/workflows/test.yml | 6 ++- .travis.dist.yml | 3 ++ .travis.yml | 5 ++- Makefile | 3 +- docs/CHANGELOG.md | 6 ++- docs/CodeCoverage.md | 2 + docs/GHAFileExplained.md | 4 +- gha.dist.yml | 1 + src/Command/PHPUnitCommand.php | 80 ++++++++++++++++++++++++++++++++-- 9 files changed, 97 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e9abe84..a673ed9b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,8 @@ jobs: uses: shivammathur/setup-php@v2 with: php-version: 7.3 - coverage: none + # Change this to pcov once we bump to PHPUnit 8.x. + coverage: xdebug - name: Initialise run: make init @@ -63,7 +64,8 @@ jobs: with: php-version: ${{ matrix.php }} extensions: pgsql, zip, gd, xmlrpc, soap - coverage: none + # Change this to pcov once 310_STABLE (PHPUnit 8.x) is the min version. + coverage: xdebug - name: Initialise moodle-plugin-ci run: | diff --git a/.travis.dist.yml b/.travis.dist.yml index 443dd51f..9d865ffc 100644 --- a/.travis.dist.yml +++ b/.travis.dist.yml @@ -28,7 +28,10 @@ env: before_install: - if [[ ${TRAVIS_PHP_VERSION:0:1} -gt 7 ]]; then pecl install xmlrpc-beta; fi - echo 'max_input_vars=5000' >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini + # Don't disable xdebug if you want to get it used for code coverage. - phpenv config-rm xdebug.ini + # Alternative (for Moodle 3.10 and up) is to use pcov. It can be installed using: + # - pecl install pcov - cd ../.. - composer create-project -n --no-dev --prefer-dist moodlehq/moodle-plugin-ci ci ^3 - export PATH="$(cd ci/bin; pwd):$(cd ci/vendor/bin; pwd):$PATH" diff --git a/.travis.yml b/.travis.yml index f7816d98..8d6de63b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,8 @@ env: - MOODLE_BRANCH=master before_install: - - phpenv config-rm xdebug.ini + # We want to verify that xdebug works of for coverage. Once we only support Moodle 3.10 and up, can switch our tests to pcov too. + # - phpenv config-rm xdebug.ini - make init # Mimic how a Moodle plugin would be run. - cp -R tests/Fixture/moodle-local_ci ../moodle-local_ci @@ -59,7 +60,7 @@ script: "$MOODLE_BRANCH" != 'MOODLE_310_STABLE' -a \ "$MOODLE_BRANCH" != 'MOODLE_39_STABLE' ] - moodle-plugin-ci phpdoc - - moodle-plugin-ci phpunit --coverage-text --fail-on-warning + - moodle-plugin-ci phpunit --verbose --coverage-text --fail-on-warning - moodle-plugin-ci behat --profile default - moodle-plugin-ci behat --profile chrome diff --git a/Makefile b/Makefile index 9578ac88..2f392805 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,7 @@ test-phpunit: check-init validate: check-init psalm check-docs $(FIXER) fix --dry-run --stop-on-violation $(COMPOSER) validate - phpdbg --version - phpdbg -d memory_limit=-1 -qrr $(PHPUNIT) --coverage-text + XDEBUG_MODE=coverage $(PHPUNIT) --verbose --coverage-text .PHONY:build build: build/moodle-plugin-ci.phar diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fa70102e..fb8aef5b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,8 +9,10 @@ This project adheres to [Semantic Versioning](http://semver.org/). The format of this change log follows the advice given at [Keep a CHANGELOG](http://keepachangelog.com). ## [Unreleased] -### Changed -- It is possible to specify more test execution options to the 'phpunit' command, such as "--fail-on-warning" +### Added +- PHPUnit code coverage will now automatically fallback between `pcov` => `xdebug` => `phpdbg`, using the "best" one available in the system. Still, if needed to, any of them can be forced, given all their requirements are fulfilled, using the following new options of the 'phpunit' command: `--coverage-pcov`, `--coverage-xdebug` or `--coverage-phpdbg`. +- ACTION SUGGESTED: Ensure that the `pcov` or `xdebug` extensions are installed in your environment to get 'moodle-plugin-ci' using them automatically. +- It is possible to specify more test execution options to the 'phpunit' command, such as `--fail-on-incomplete`, `--fail-on-risky` and `--fail-on-skipped` and `--fail-on-warning`. For more information, see [PHPUnit documentation](https://phpunit.readthedocs.io). ## [3.2.5] - 2022-03-31 ### Changed diff --git a/docs/CodeCoverage.md b/docs/CodeCoverage.md index d6b73e09..437dffe2 100644 --- a/docs/CodeCoverage.md +++ b/docs/CodeCoverage.md @@ -6,6 +6,8 @@ title: Generating code coverage Currently, code coverage is only generated for builds that are running on PHP7 or later. Code coverage generation is significantly faster and easier to produce in PHP7. +Code coverage will now automatically fallback between `pcov` => `xdebug` => `phpdbg`, using the "best" one available in the system. Still, if needed to, any of them can be forced, given all their requirements are fulfilled, using the following new options of the 'phpunit' command: `--coverage-pcov`, `--coverage-xdebug` or `--coverage-phpdbg`. + The way you generate code coverage is to use one of the coverage options on the `phpunit` command. The currently available options are `--coverage-text` and `--coverage-clover`. The easiest way to start generating code coverage is to use the text option as that gets printed in the Travis CI logs. Example: diff --git a/docs/GHAFileExplained.md b/docs/GHAFileExplained.md index 762f387b..590b61e5 100644 --- a/docs/GHAFileExplained.md +++ b/docs/GHAFileExplained.md @@ -54,7 +54,8 @@ jobs: # another branch, total number of builds will become 12. # If you need to use PHP 7.0 and run phpunit coverage test, make sure you are # using ubuntu-16.04 virtual environment in this case to have phpdbg or - # this version in the system. + # this version in the system. See the "Setup PHP" step below for more details + # about PHPUnit code coverage. strategy: fail-fast: false matrix: @@ -94,6 +95,7 @@ jobs: php-version: ${{ matrix.php }} extensions: ${{ matrix.extensions }} ini-values: max_input_vars=5000 + # none to use phpdbg fallback. Specify pcov (Moodle 3.10 and up) or xdebug to use them instead. coverage: none # Install this project into a directory called "ci", updating PATH and diff --git a/gha.dist.yml b/gha.dist.yml index a8968e2d..e595888f 100644 --- a/gha.dist.yml +++ b/gha.dist.yml @@ -46,6 +46,7 @@ jobs: php-version: ${{ matrix.php }} extensions: ${{ matrix.extensions }} ini-values: max_input_vars=5000 + # none to use phpdbg fallback. Specify pcov (Moodle 3.10 and up) or xdebug to use them instead. coverage: none - name: Initialise moodle-plugin-ci diff --git a/src/Command/PHPUnitCommand.php b/src/Command/PHPUnitCommand.php index bda01704..575c8139 100644 --- a/src/Command/PHPUnitCommand.php +++ b/src/Command/PHPUnitCommand.php @@ -31,6 +31,9 @@ protected function configure() ->setDescription('Run PHPUnit on a plugin') ->addOption('coverage-text', null, InputOption::VALUE_NONE, 'Generate and print code coverage report in text format') ->addOption('coverage-clover', null, InputOption::VALUE_NONE, 'Generate code coverage report in Clover XML format') + ->addOption('coverage-pcov', null, InputOption::VALUE_NONE, 'Use the pcov extension to calculate code coverage') + ->addOption('coverage-xdebug', null, InputOption::VALUE_NONE, 'Use the xdebug extension to calculate code coverage') + ->addOption('coverage-phpdbg', null, InputOption::VALUE_NONE, 'Use the phpdbg binary to calculate code coverage') ->addOption('fail-on-incomplete', null, InputOption::VALUE_NONE, 'Treat incomplete tests as failures') ->addOption('fail-on-risky', null, InputOption::VALUE_NONE, 'Treat risky tests as failures') ->addOption('fail-on-skipped', null, InputOption::VALUE_NONE, 'Treat skipped tests as failures') @@ -52,7 +55,7 @@ protected function execute(InputInterface $input, OutputInterface $output) } $colors = $output->isDecorated() ? '--colors="always"' : ''; - $binary = $this->resolveBinary($input); + $binary = $this->resolveBinary($input, $output); $options = $this->resolveOptions($input); $process = $this->execute->passThrough( sprintf('%s%s/vendor/bin/phpunit %s %s', $binary, $this->moodle->directory, $colors, $options), @@ -78,6 +81,9 @@ private function resolveOptions(InputInterface $input) if ($this->supportsCoverage() && $input->getOption('coverage-clover')) { $options[] = sprintf('--coverage-clover %s/coverage.xml', getcwd()); } + if ($input->getOption('verbose')) { + $options[] = '--verbose'; + } foreach (['fail-on-warning', 'fail-on-risky', 'fail-on-skipped', 'fail-on-warning'] as $option) { if ($input->getOption($option)) { $options[] = '--'.$option; @@ -95,11 +101,12 @@ private function resolveOptions(InputInterface $input) /** * Use phpdbg if we are generating code coverage. * - * @param InputInterface $input + * @param InputInterface $input + * @param OutputInterface $output * * @return string */ - private function resolveBinary(InputInterface $input) + private function resolveBinary(InputInterface $input, OutputInterface $output) { if (!$this->supportsCoverage()) { return ''; @@ -108,7 +115,19 @@ private function resolveBinary(InputInterface $input) return ''; } - return 'phpdbg -d memory_limit=-1 -qrr '; + // Depending on the coverage driver, selected return different values. + switch ($this->resolveCoverageDriver($input, $output)) { + case 'pcov': + return 'php -dxdebug.mode=off -dpcov.enabled=1 -dpcov.directory=. '; // Enable pcov, disable xdebug, just in case. + case 'xdebug': + return 'php -dpcov.enabled=0 -dxdebug.mode=coverage '; // Enable xdebug, disable pcov, just in case. + case 'phpdbg': + return 'phpdbg -d memory_limit=-1 -qrr '; + } + // No suitable coverage driver found, disabling all candidates. + $output->writeln('No suitable driver found, disabling code coverage.'); + + return 'php -dpcov.enabled=0 -dxdebug.mode=off '; } /** @@ -120,4 +139,57 @@ private function supportsCoverage() { return version_compare(PHP_VERSION, '7.0.0', '>='); } + + /** + * Given the current environment and options return the code coverage driver to use. + * + * @param InputInterface $input + * @param OutputInterface $output + * + * @return string one of pcov, xdebug, phpdbg + */ + private function resolveCoverageDriver(InputInterface $input, OutputInterface $output) + { + // Let's see if any of the coverage drivers has been forced via command line options. + if ($input->getOption('coverage-pcov')) { + // Before accepting it, perform some checks and report. + if (!extension_loaded('pcov')) { + $output->writeln('PHP pcov extension not available.'); + + return ''; + } + if ($this->moodle->getBranch() < 310) { + $output->writeln('PHP pcov coverage only can be used with Moodle 3.10 and up.'); + + return ''; + } + + return 'pcov'; + } elseif ($input->getOption('coverage-xdebug')) { + // Before accepting it, perform some checks and report. + if (!extension_loaded('xdebug')) { + $output->writeln('PHP xdebug extension not available.'); + + return ''; + } + + return 'xdebug'; + } elseif ($input->getOption('coverage-phpdbg')) { + return 'phpdbg'; + } + + // Arrived here, let's find the best (pcov => xdebug => phpdbg) available driver. + + if (extension_loaded('pcov') && $this->moodle->getBranch() >= 310) { + // If pcov is available and we are using Moodle 3.10 (PHPUnit 8.5) and up, let's use it. + return 'pcov'; + } + + if (extension_loaded('xdebug')) { + // If xdebug is available, let's use it. + return 'xdebug'; + } + + return 'phpdbg'; // Fallback to phpdbg (bundled with php 7.0 and up) if none of the above are available. + } }