-
Notifications
You must be signed in to change notification settings - Fork 13.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Build] Collect frontend code coverage from Cypress tests #9555
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,10 +15,14 @@ | |
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
# | ||
set -e | ||
|
||
GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-.} | ||
ASSETS_MANIFEST="$GITHUB_WORKSPACE/superset/static/assets/manifest.json" | ||
|
||
# Echo only when not in parallel mode | ||
say() { | ||
if [[ ${INPUT_PARALLEL^^} != 'TRUE' ]]; then | ||
if [[ $(echo "$INPUT_PARALLEL" | tr '[:lower:]' '[:upper:]') != 'TRUE' ]]; then | ||
echo "$1" | ||
fi | ||
} | ||
|
@@ -67,41 +71,27 @@ build-assets() { | |
say "::endgroup::" | ||
} | ||
|
||
npm-build() { | ||
if [[ $1 = '--no-cache' ]]; then | ||
build-assets | ||
build-assets-cached() { | ||
cache-restore assets | ||
if [[ -f "$ASSETS_MANIFEST" ]]; then | ||
echo 'Skip frontend build because static assets already exist.' | ||
else | ||
cache-restore assets | ||
if [[ -f $GITHUB_WORKSPACE/superset/static/assets/manifest.json ]]; then | ||
echo 'Skip frontend build because static assets already exist.' | ||
else | ||
build-assets | ||
cache-save assets | ||
fi | ||
build-assets | ||
cache-save assets | ||
fi | ||
} | ||
|
||
cypress-install() { | ||
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base" | ||
|
||
cache-restore cypress | ||
|
||
say "::group::Install Cypress" | ||
npm ci | ||
say "::endgroup::" | ||
|
||
cache-save cypress | ||
} | ||
build-instrumented-assets() { | ||
cd "$GITHUB_WORKSPACE/superset-frontend" | ||
|
||
testdata() { | ||
cd "$GITHUB_WORKSPACE" | ||
say "::group::Load test data" | ||
# must specify PYTHONPATH to make `tests.superset_test_config` importable | ||
export PYTHONPATH="$GITHUB_WORKSPACE" | ||
superset db upgrade | ||
superset load_test_users | ||
superset load_examples --load-test-data | ||
superset init | ||
say "::group::Build static assets with JS instrumented for test coverage" | ||
cache-restore instrumented-assets | ||
if [[ -f "$ASSETS_MANIFEST" ]]; then | ||
echo 'Skip frontend build because instrumented static assets already exist.' | ||
else | ||
npm run build-instrumented -- --no-progress | ||
cache-save instrumented-assets | ||
fi | ||
say "::endgroup::" | ||
} | ||
|
||
|
@@ -131,3 +121,100 @@ setup-mysql() { | |
EOF | ||
say "::endgroup::" | ||
} | ||
|
||
testdata() { | ||
cd "$GITHUB_WORKSPACE" | ||
say "::group::Load test data" | ||
# must specify PYTHONPATH to make `tests.superset_test_config` importable | ||
export PYTHONPATH="$GITHUB_WORKSPACE" | ||
superset db upgrade | ||
superset load_test_users | ||
superset load_examples --load-test-data | ||
superset init | ||
say "::endgroup::" | ||
} | ||
|
||
codecov() { | ||
say "::group::Upload code coverage" | ||
local codecovScript="${HOME}/codecov.sh" | ||
# download bash script if needed | ||
if [[ ! -f "$codecovScript" ]]; then | ||
curl -s https://codecov.io/bash > "$codecovScript" | ||
fi | ||
bash "$codecovScript" "$@" | ||
say "::endgroup::" | ||
} | ||
|
||
cypress-install() { | ||
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base" | ||
|
||
cache-restore cypress | ||
|
||
say "::group::Install Cypress" | ||
npm ci | ||
say "::endgroup::" | ||
|
||
cache-save cypress | ||
} | ||
|
||
# Run Cypress and upload coverage reports | ||
cypress-run() { | ||
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base" | ||
|
||
local page=$1 | ||
local group=${2:-Default} | ||
local cypress="./node_modules/.bin/cypress run" | ||
local browser=${CYPRESS_BROWSER:-chrome} | ||
|
||
say "::group::Run Cypress for [$page]" | ||
if [[ -z $CYPRESS_RECORD_KEY ]]; then | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Now we only record Cypress runs when |
||
$cypress --spec "cypress/integration/$page" --browser "$browser" | ||
else | ||
# additional flags for Cypress dashboard recording | ||
$cypress --spec "cypress/integration/$page" --browser "$browser" --record \ | ||
--group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Must add quotes to allow spaces in parameters. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. BTW, are you using shellcheck to lint these scripts? I recommend it:
You could even add a lint scripts CI step ;) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just tried it out and it works great! Previously I was relying on Codacy. |
||
fi | ||
|
||
# don't add quotes to $record because we do want word splitting | ||
say "::endgroup::" | ||
} | ||
|
||
cypress-run-all() { | ||
# Start Flask and run it in background | ||
# --no-debugger means disable the interactive debugger on the 500 page | ||
# so errors can print to stderr. | ||
local flasklog="${HOME}/flask.log" | ||
local port=8081 | ||
|
||
nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null & | ||
local flaskProcessId=$! | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
|
||
cypress-run "*/*" | ||
|
||
# Upload code coverage separately so each page can have separate flags | ||
# -c will clean existing coverage reports, -F means add flags | ||
codecov -cF "cypress" | ||
|
||
# After job is done, print out Flask log for debugging | ||
say "::group::Flask log for default run" | ||
cat "$flasklog" | ||
say "::endgroup::" | ||
|
||
# Rerun SQL Lab tests with backend persist enabled | ||
export SUPERSET_CONFIG=tests.superset_test_config_sqllab_backend_persist | ||
|
||
# Restart Flask with new configs | ||
kill $flaskProcessId | ||
nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null & | ||
local flaskProcessId=$! | ||
|
||
cypress-run "sqllab/*" "Backend persist" | ||
codecov -cF "cypress" | ||
|
||
say "::group::Flask log for backend persist" | ||
cat "$flasklog" | ||
say "::endgroup::" | ||
|
||
# make sure the program exits | ||
kill $flaskProcessId | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -21,6 +21,17 @@ | |
const workspaceDirectory = process.env.GITHUB_WORKSPACE; | ||
const homeDirectory = process.env.HOME; | ||
|
||
const assetsConfig = { | ||
path: [`${workspaceDirectory}/superset/static/assets`], | ||
hashFiles: [ | ||
`${workspaceDirectory}/superset-frontend/src/**/*`, | ||
`${workspaceDirectory}/superset-frontend/*.js`, | ||
`${workspaceDirectory}/superset-frontend/*.json`, | ||
], | ||
// dont use restore keys as it may give an invalid older build | ||
restoreKeys: '', | ||
}; | ||
|
||
// Multi-layer cache definition | ||
module.exports = { | ||
pip: { | ||
|
@@ -31,18 +42,11 @@ module.exports = { | |
path: [`${homeDirectory}/.npm`], | ||
hashFiles: ['superset-frontend/package-lock.json'], | ||
}, | ||
assets: { | ||
path: [ | ||
`${workspaceDirectory}/superset/static/assets`, | ||
], | ||
hashFiles: [ | ||
`${workspaceDirectory}/superset-frontend/src/**/*`, | ||
`${workspaceDirectory}/superset-frontend/*.json`, | ||
`${workspaceDirectory}/superset-frontend/*.js`, | ||
], | ||
// dont use restore keys as it may give an invalid older build | ||
restoreKeys: '' | ||
}, | ||
assets: assetsConfig, | ||
// use separate cache for instrumented JS files and regular assets | ||
// one is built with `npm run build`, | ||
// another is built with `npm run build-instrumented` | ||
'instrumented-assets': assetsConfig, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use separate cache keys for regular assets and instrumented JS files that supports code coverage. The regular assets are not currently in use, though. We can potentially add a sanity check in |
||
cypress: { | ||
path: [`${homeDirectory}/.cache/Cypress`], | ||
hashFiles: [ | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,7 +7,7 @@ jobs: | |
name: Cypress | ||
runs-on: ubuntu-18.04 | ||
strategy: | ||
fail-fast: false | ||
fail-fast: true | ||
matrix: | ||
browser: ['chrome'] | ||
env: | ||
|
@@ -17,7 +17,6 @@ jobs: | |
postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset | ||
PYTHONPATH: ${{ github.workspace }} | ||
REDIS_PORT: 16379 | ||
CI: github-actions | ||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
services: | ||
postgres: | ||
|
@@ -40,42 +39,19 @@ jobs: | |
python-version: '3.6' | ||
|
||
- name: Install dependencies | ||
uses: apache-superset/cached-dependencies@ddf7d7f | ||
uses: apache-superset/cached-dependencies@adc6f73 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This upgrade fixes a bug related to cache validation. |
||
with: | ||
# Run commands in parallel does help initial installation without cache | ||
parallel: true | ||
run: | | ||
npm-install && npm-build | ||
npm-install && build-instrumented-assets | ||
pip-install && setup-postgres && testdata | ||
cypress-install | ||
|
||
- name: Cypress run all | ||
- name: Run Cypress | ||
uses: apache-superset/cached-dependencies@adc6f73 | ||
env: | ||
CYPRESS_GROUP: Default | ||
CYPRESS_PATH: 'cypress/integration/*/*' | ||
run: | | ||
# Start Flask and run Cypress | ||
|
||
# --no-debugger means disable the interactive debugger on the 500 page | ||
# so errors can print to stderr. | ||
flask run --no-debugger --with-threads -p 8081 & | ||
|
||
sleep 3 # wait for the Flask app to start | ||
|
||
cd ${{ github.workspace }}/superset-frontend/cypress-base/ | ||
npm run cypress -- run --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}" --record false | ||
|
||
- name: Cypress run SQL Lab (with backend persist) | ||
env: | ||
SUPERSET_CONFIG: tests.superset_test_config_sqllab_backend_persist | ||
CYPRESS_GROUP: Backend persist | ||
CYPRESS_PATH: 'cypress/integration/sqllab/*' | ||
run: | | ||
# Start Flask with alternative config and run Cypress | ||
|
||
killall python # exit the running Flask app | ||
flask run --no-debugger --with-threads -p 8081 & | ||
sleep 3 # wait for the Flask app to start | ||
|
||
cd ${{ github.workspace }}/superset-frontend/cypress-base/ | ||
npm run cypress -- run --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}" --record false | ||
CYPRESS_BROWSER: ${{ matrix.browser }} | ||
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} | ||
with: | ||
run: cypress-run-all | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Moved to |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Added a new
build-instrumented
npm script to build instrumented JS for Cypress: https://docs.cypress.io/guides/tooling/code-coverage.html#Instrumenting-code