Skip to content
This repository has been archived by the owner on Feb 20, 2023. It is now read-only.

End-to-end (JUnit and oltpbench) only for coverage #1629

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

End-to-end (JUnit and oltpbench) only for coverage #1629

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
975b498
e2e tests for coverage only.
mbutrovich Jul 23, 2021
b775f60
Don't build unit tests.
mbutrovich Jul 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
202 changes: 101 additions & 101 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,124 +19,124 @@ pipeline {
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageGithub() } }
post { cleanup { deleteDir() } }
}
stage('Check') {
parallel {
stage('ubuntu-20.04/gcc-9.3 (Debug/format/lint/censored)') {
agent { docker { image 'noisepage:focal' } }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } }
post { cleanup { deleteDir() } }
}
stage('ubuntu-20.04/clang-8.0 (Debug/format/lint/censored)') {
agent { docker { image 'noisepage:focal' } }
environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } }
post { cleanup { deleteDir() } }
}
}
}

stage('Build-only checks') {
parallel {
stage('Benchmarks (debug build only)') {
agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([
buildCommand: 'ninja',
cmake: '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_BENCHMARKS=ON',
] ) } }
post { cleanup { deleteDir() } }
}
stage('Logging disabled (release build only)') {
agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([
buildCommand: 'ninja',
cmake: '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_BENCHMARKS=ON -DNOISEPAGE_BUILD_TESTS=ON -NOISEPAGE_BUILD_SELF_DRIVING_E2E_TESTS=ON -DNOISEPAGE_USE_LOGGING=OFF'
] ) } }
post { cleanup { deleteDir() } }
}
}
}
// stage('Check') {
// parallel {
// stage('ubuntu-20.04/gcc-9.3 (Debug/format/lint/censored)') {
// agent { docker { image 'noisepage:focal' } }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } }
// post { cleanup { deleteDir() } }
// }
// stage('ubuntu-20.04/clang-8.0 (Debug/format/lint/censored)') {
// agent { docker { image 'noisepage:focal' } }
// environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } }
// post { cleanup { deleteDir() } }
// }
// }
// }
//
// stage('Build-only checks') {
// parallel {
// stage('Benchmarks (debug build only)') {
// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([
// buildCommand: 'ninja',
// cmake: '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_BENCHMARKS=ON',
// ] ) } }
// post { cleanup { deleteDir() } }
// }
// stage('Logging disabled (release build only)') {
// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([
// buildCommand: 'ninja',
// cmake: '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_BENCHMARKS=ON -DNOISEPAGE_BUILD_TESTS=ON -NOISEPAGE_BUILD_SELF_DRIVING_E2E_TESTS=ON -DNOISEPAGE_USE_LOGGING=OFF'
// ] ) } }
// post { cleanup { deleteDir() } }
// }
// }
// }

stage('Test') {
parallel {
// The first argument to utils.stageTest() indicates whether pipeline metrics should be gathered.
// Pipeline metrics take a while to run and don't need to be run in every stage. The pipeline metrics are only run in one arbitrary stage to check for breakage.

stage('ubuntu-20.04/gcc-9.3 (Debug/ASAN/jumbotests)') {
agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(true, [cmake:
'-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON'
] ) } }
post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
}
// stage('ubuntu-20.04/gcc-9.3 (Debug/ASAN/jumbotests)') {
// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(true, [cmake:
// '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON'
// ] ) } }
// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
// }

stage('ubuntu-20.04/gcc-9.3 (Debug/Coverage/unittest)') {
agent { docker { image 'noisepage:focal' ; label 'dgb' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
environment { CODECOV_TOKEN=credentials('codecov-token') }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake:
// Note that unity builds mess with coverage.
'-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_GENERATE_COVERAGE=ON'
'-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=OFF -DNOISEPAGE_GENERATE_COVERAGE=ON'
] ) } }
post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
}

stage('ubuntu-20.04/clang-8.0 (Debug/ASAN/jumbotests)') {
agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake:
'-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON'
] ) } }
post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
}

stage('ubuntu-20.04/gcc-9.3 (Release/jumbotests)') {
agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake:
'-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON'
] ) } }
post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
}

stage('ubuntu-20.04/clang-8.0 (Release/jumbotests)') {
agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake:
'-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON'
] ) } }
post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
}
}
}

stage('End-to-End') {
parallel {
stage('Debug') {
agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageOltpbenchDebug() } }
post { cleanup { deleteDir() } }
}
stage('Performance') {
agent { label 'benchmark' }
environment { PSS_CREATOR = credentials('pss-creator') /* Performance Storage Service (Django) auth credentials. Can only be changed from Jenkins webpage. */ }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageOltpbenchRelease() } }
post { cleanup { deleteDir() } }
}
//
// stage('ubuntu-20.04/clang-8.0 (Debug/ASAN/jumbotests)') {
// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
// environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake:
// '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON'
// ] ) } }
// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
// }
//
// stage('ubuntu-20.04/gcc-9.3 (Release/jumbotests)') {
// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake:
// '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON'
// ] ) } }
// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
// }
//
// stage('ubuntu-20.04/clang-8.0 (Release/jumbotests)') {
// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
// environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake:
// '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON'
// ] ) } }
// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
// }
}
}

stage('Self-Driving') {
parallel {
stage('Workload Forecasting'){
agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageForecasting() } }
post { cleanup { deleteDir() } }
}
stage('Modeling'){
agent { docker { image 'noisepage:focal' ; label 'dgb' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
environment { CODECOV_TOKEN=credentials('codecov-token') }
steps { script { utils = utils ?: load(utilsFileName) ; utils.stageModeling() } }
post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
}
}
}
// stage('End-to-End') {
// parallel {
// stage('Debug') {
// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageOltpbenchDebug() } }
// post { cleanup { deleteDir() } }
// }
// stage('Performance') {
// agent { label 'benchmark' }
// environment { PSS_CREATOR = credentials('pss-creator') /* Performance Storage Service (Django) auth credentials. Can only be changed from Jenkins webpage. */ }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageOltpbenchRelease() } }
// post { cleanup { deleteDir() } }
// }
// }
// }
//
// stage('Self-Driving') {
// parallel {
// stage('Workload Forecasting'){
// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageForecasting() } }
// post { cleanup { deleteDir() } }
// }
// stage('Modeling'){
// agent { docker { image 'noisepage:focal' ; label 'dgb' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } }
// environment { CODECOV_TOKEN=credentials('codecov-token') }
// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageModeling() } }
// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } }
// }
// }
// }
}
}
55 changes: 45 additions & 10 deletions Jenkinsfile-utils.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,18 +81,53 @@ void stageTest(Boolean runPipelineMetrics, Map args = [:]) {
sh script: "cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended", label: 'UnitTest (Extended)'
sh script: "cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended -a 'compiled_query_execution=True' -a 'bytecode_handlers_path=./bytecode_handlers_ir.bc'", label: 'UnitTest (Extended, Compiled Execution)'

if (runPipelineMetrics) {
sh script: "cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended -a 'pipeline_metrics_enable=True' -a 'pipeline_metrics_sample_rate=100' -a 'counters_enable=True' -a 'query_trace_metrics_enable=True'", label: 'UnitTest (Extended with pipeline metrics, counters, and query trace metrics)'
sh script: "cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended -a 'pipeline_metrics_enable=True' -a 'pipeline_metrics_sample_rate=100' -a 'counters_enable=True' -a 'query_trace_metrics_enable=True' -a 'compiled_query_execution=True' -a 'bytecode_handlers_path=./bytecode_handlers_ir.bc'", label: 'UnitTest (Extended, Compiled Execution with pipeline metrics, counters, and query trace metrics)'
}
// if (runPipelineMetrics) {
// sh script: "cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended -a 'pipeline_metrics_enable=True' -a 'pipeline_metrics_sample_rate=100' -a 'counters_enable=True' -a 'query_trace_metrics_enable=True'", label: 'UnitTest (Extended with pipeline metrics, counters, and query trace metrics)'
// sh script: "cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended -a 'pipeline_metrics_enable=True' -a 'pipeline_metrics_sample_rate=100' -a 'counters_enable=True' -a 'query_trace_metrics_enable=True' -a 'compiled_query_execution=True' -a 'bytecode_handlers_path=./bytecode_handlers_ir.bc'", label: 'UnitTest (Extended, Compiled Execution with pipeline metrics, counters, and query trace metrics)'
// }
//
// sh 'cd build && timeout 1h ninja check-tpl'
//
// if (args.cmake.toUpperCase().contains("NOISEPAGE_USE_JUMBOTESTS=ON")) {
// sh 'cd build && export BUILD_ABS_PATH=`pwd` && timeout 1h ninja jumbotests'
// } else {
// sh 'cd build && export BUILD_ABS_PATH=`pwd` && timeout 1h ninja unittest'
// }

sh 'cd build && timeout 1h ninja check-tpl'
sh script: '''
cd build
PYTHONPATH=.. timeout 10m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/tatp.json --build-type=debug
''', label:'OLTPBench (TATP)'

if (args.cmake.toUpperCase().contains("NOISEPAGE_USE_JUMBOTESTS=ON")) {
sh 'cd build && export BUILD_ABS_PATH=`pwd` && timeout 1h ninja jumbotests'
} else {
sh 'cd build && export BUILD_ABS_PATH=`pwd` && timeout 1h ninja unittest'
}
sh script: '''
cd build
PYTHONPATH=.. timeout 10m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/tatp_wal_disabled.json --build-type=debug
''', label: 'OLTPBench (No WAL)'

sh script: '''
cd build
PYTHONPATH=.. timeout 10m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/smallbank.json --build-type=debug
''', label:'OLTPBench (Smallbank)'

sh script: '''
cd build
PYTHONPATH=.. timeout 10m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/ycsb.json --build-type=debug
''', label: 'OLTPBench (YCSB)'

sh script: '''
cd build
PYTHONPATH=.. timeout 5m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/noop.json --build-type=debug
''', label: 'OLTPBench (NOOP)'

sh script: '''
cd build
PYTHONPATH=.. timeout 30m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/tpcc.json --build-type=debug
''', label: 'OLTPBench (TPCC)'

sh script: '''
cd build
PYTHONPATH=.. timeout 30m python3 -m script.testing.oltpbench --config-file=../script/testing/oltpbench/configs/end_to_end_debug/tpcc_parallel_disabled.json --build-type=debug
''', label: 'OLTPBench (No Parallel)'

if (args.cmake.toUpperCase().contains("NOISEPAGE_GENERATE_COVERAGE=ON")) {
uploadCoverage()
Expand Down