User-Defined Functions

.gitignore

@@ -48,8 +48,8 @@ config/
 configure
 config-h.in
 autom4te.cache
-*Makefile.in
-*Makefile
+build/*Makefile.in
+build/*Makefile
 libtool
 aclocal.m4
 config.log

CMakeLists.txt

@@ -868,7 +868,7 @@ file(GLOB_RECURSE
 function(add_test_util_lib TYPE)
     string(TOLOWER ${TYPE} TYPE_LOWER)
     add_library(noisepage_test_util_${TYPE_LOWER} ${TYPE} ${NOISEPAGE_TEST_UTIL_SRCS})
-    add_custom_command(TARGET noisepage_test_util_${TYPE_LOWER} DEPENDS gtest gtest_main gmock gmock_main)
+    add_custom_command(TARGET noisepage_test_util_${TYPE_LOWER} DEPENDS gtest gtest_main)
     target_compile_options(noisepage_test_util_${TYPE_LOWER} PRIVATE "-Werror" "-Wall")
     # Inject the source directory path into the translation units for test utility lib
     target_compile_definitions(noisepage_test_util_${TYPE_LOWER} PRIVATE NOISEPAGE_BUILD_ROOT=${CMAKE_BINARY_DIR})
@@ -878,7 +878,7 @@ function(add_test_util_lib TYPE)
             ${CMAKE_BINARY_DIR}/_deps/src/googletest/googletest/include/
             )
     target_link_libraries(noisepage_test_util_${TYPE_LOWER} PUBLIC
-            ${CMAKE_BINARY_DIR}/lib/libgtest.a ${CMAKE_BINARY_DIR}/lib/libgmock.a
+            ${CMAKE_BINARY_DIR}/lib/libgtest.a
             util_${TYPE_LOWER} pqxx)
     set_target_properties(noisepage_test_util_${TYPE_LOWER} PROPERTIES CXX_EXTENSIONS OFF UNITY_BUILD ${NOISEPAGE_UNITY_BUILD})
 endfunction()
@@ -917,7 +917,6 @@ function(add_noisepage_test
     add_executable(${TEST_NAME} ${EXCLUDE_OPTION} ${TEST_SOURCES})
 
     target_compile_options(${TEST_NAME} PRIVATE "-Werror" "-Wall" "-fvisibility=hidden")
-    target_link_libraries(${TEST_NAME} PRIVATE ${CMAKE_BINARY_DIR}/lib/libgmock_main.a)
     if (${NOISEPAGE_ENABLE_SHARED})
         target_link_libraries(${TEST_NAME} PRIVATE noisepage_test_util_shared)
     else ()
@@ -1119,10 +1118,10 @@ file(GLOB_RECURSE
         )
 
 add_library(noisepage_benchmark_util STATIC ${NOISEPAGE_BENCHMARK_UTIL_SRCS})
-add_custom_command(TARGET noisepage_benchmark_util DEPENDS gtest gtest_main gmock gmock_main)
+add_custom_command(TARGET noisepage_benchmark_util DEPENDS gtest gtest_main)
 target_compile_options(noisepage_benchmark_util PRIVATE "-Werror" "-Wall")
 target_include_directories(noisepage_benchmark_util PUBLIC ${PROJECT_SOURCE_DIR}/benchmark/include)
-target_link_libraries(noisepage_benchmark_util PUBLIC ${CMAKE_BINARY_DIR}/lib/libgmock_main.a noisepage_test_util_static benchmark)
+target_link_libraries(noisepage_benchmark_util PUBLIC noisepage_test_util_static benchmark)
 set_target_properties(noisepage_benchmark_util PROPERTIES CXX_EXTENSIONS OFF)
 
 set(NOISEPAGE_BENCHMARKS "")

Jenkinsfile-utils.groovy

@@ -76,13 +76,13 @@ void stageTest(Boolean runPipelineMetrics, Map args = [:]) {
 
     buildType = (args.cmake.toUpperCase().contains("CMAKE_BUILD_TYPE=RELEASE")) ? "release" : "debug"
 
-    sh script: "cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=$buildType --query-mode=simple", label: 'UnitTest (Simple)'
+    sh script: "cd build && PYTHONPATH=.. timeout 40m python3 -m script.testing.junit --build-type=$buildType --query-mode=simple", label: 'UnitTest (Simple)'
     sh script: "cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=$buildType --query-mode=simple -a 'compiled_query_execution=True' -a 'bytecode_handlers_path=./bytecode_handlers_ir.bc'", label: 'UnitTest (Simple, Compiled Execution)'
-    sh script: "cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended", label: 'UnitTest (Extended)'
+    sh script: "cd build && PYTHONPATH=.. timeout 40m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended", label: 'UnitTest (Extended)'
     sh script: "cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended -a 'compiled_query_execution=True' -a 'bytecode_handlers_path=./bytecode_handlers_ir.bc'", label: 'UnitTest (Extended, Compiled Execution)'
 
     if (runPipelineMetrics) {
-        sh script: "cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended -a 'pipeline_metrics_enable=True' -a 'pipeline_metrics_sample_rate=100' -a 'counters_enable=True' -a 'query_trace_metrics_enable=True'", label: 'UnitTest (Extended with pipeline metrics, counters, and query trace metrics)'
+        sh script: "cd build && PYTHONPATH=.. timeout 40m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended -a 'pipeline_metrics_enable=True' -a 'pipeline_metrics_sample_rate=100' -a 'counters_enable=True' -a 'query_trace_metrics_enable=True'", label: 'UnitTest (Extended with pipeline metrics, counters, and query trace metrics)'
         sh script: "cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=$buildType --query-mode=extended -a 'pipeline_metrics_enable=True' -a 'pipeline_metrics_sample_rate=100' -a 'counters_enable=True' -a 'query_trace_metrics_enable=True' -a 'compiled_query_execution=True' -a 'bytecode_handlers_path=./bytecode_handlers_ir.bc'", label: 'UnitTest (Extended, Compiled Execution with pipeline metrics, counters, and query trace metrics)'
     }
 

Makefile

@@ -0,0 +1,20 @@
+# Makefile
+# Some scripting shortcuts.
+
+# Run all regression checks in all execution modes
+check-regress: check-regress-interpreted check-regress-compiled
+
+# Run all regression tests in interpreted mode
+check-regress-interpreted:
+	cd build && PYTHONPATH=.. python -m script.testing.junit --build-type=debug --query-mode=simple
+
+# Run all regression tests in compiled mode
+check-regress-compiled:
+	PYTHONPATH=.. python -m script.testing.junit --build-type=debug --query-mode=simple -a 'compiled_query_execution=True' -a 'bytecode_handlers_path=./bytecode_handlers_ir.bc'
+
+check-regress-udf:
+	cd build && PYTHONPATH=.. python -m script.testing.junit --build-type=debug --query-mode=simple --tracefile-test=udf.test
+
+# Re-generate the trace file for UDF regression tests
+generate-regress-udf:
+	cd script/testing/junit/ && ant generate-trace -Dpath=sql/udf.sql -Ddb-url=jdbc:postgresql://localhost/test -Ddb-user=postgres -Ddb-password=password -Doutput-name=udf.test

benchmark/runner/execution_runners.cpp

@@ -11,6 +11,7 @@
 #include "common/scoped_timer.h"
 #include "execution/compiler/compilation_context.h"
 #include "execution/compiler/executable_query.h"
+#include "execution/exec/execution_context_builder.h"
 #include "execution/exec/execution_settings.h"
 #include "execution/execution_util.h"
 #include "execution/sql/ddl_executors.h"
@@ -448,9 +449,17 @@ class ExecutionRunners : public benchmark::Fixture {
       exec_settings = *exec_settings_arg;
     }
 
-    auto exec_ctx = std::make_unique<execution::exec::ExecutionContext>(
-        db_oid, common::ManagedPointer(txn), execution::exec::NoOpResultConsumer(), out_plan->GetOutputSchema().Get(),
-        common::ManagedPointer(accessor), exec_settings, metrics_manager_, DISABLED, DISABLED);
+    auto exec_ctx = execution::exec::ExecutionContextBuilder()
+                        .WithDatabaseOID(db_oid)
+                        .WithExecutionSettings(exec_settings)
+                        .WithTxnContext(common::ManagedPointer{txn})
+                        .WithOutputSchema(out_plan->GetOutputSchema())
+                        .WithOutputCallback(execution::exec::NoOpResultConsumer{})
+                        .WithCatalogAccessor(common::ManagedPointer{accessor})
+                        .WithMetricsManager(metrics_manager_)
+                        .WithReplicationManager(DISABLED)
+                        .WithRecoveryManager(DISABLED)
+                        .Build();
 
     execution::compiler::ExecutableQuery::query_identifier.store(ExecutionRunners::query_id++);
     auto exec_query = execution::compiler::CompilationContext::Compile(*out_plan, exec_settings, accessor.get(),
@@ -494,9 +503,18 @@ class ExecutionRunners : public benchmark::Fixture {
     auto txn = txn_manager->BeginTransaction();
     auto accessor = catalog->GetAccessor(common::ManagedPointer(txn), db_oid, DISABLED);
     auto exec_settings = ExecutionRunners::GetExecutionSettings();
-    auto exec_ctx = std::make_unique<execution::exec::ExecutionContext>(
-        db_oid, common::ManagedPointer(txn), nullptr, nullptr, common::ManagedPointer(accessor), exec_settings,
-        metrics_manager_, DISABLED, DISABLED);
+
+    auto exec_ctx = execution::exec::ExecutionContextBuilder()
+                        .WithDatabaseOID(db_oid)
+                        .WithExecutionSettings(exec_settings)
+                        .WithTxnContext(common::ManagedPointer{txn})
+                        .WithOutputSchema(execution::exec::ExecutionContext::NULL_OUTPUT_SCHEMA)
+                        .WithOutputCallback(execution::exec::ExecutionContext::NULL_OUTPUT_CALLBACK)
+                        .WithCatalogAccessor(common::ManagedPointer{accessor})
+                        .WithMetricsManager(metrics_manager_)
+                        .WithReplicationManager(DISABLED)
+                        .WithRecoveryManager(DISABLED)
+                        .Build();
 
     execution::sql::TableGenerator table_generator(exec_ctx.get(), block_store, accessor->GetDefaultNamespace());
     if (is_build) {
@@ -529,12 +547,12 @@ class ExecutionRunners : public benchmark::Fixture {
   }
 
   void BenchmarkExecQuery(int64_t num_iters, execution::compiler::ExecutableQuery *exec_query,
-                          planner::OutputSchema *out_schema, bool commit,
+                          const planner::OutputSchema *out_schema, bool commit,
                           std::vector<std::vector<parser::ConstantValueExpression>> *params = &empty_params,
                           execution::exec::ExecutionSettings *exec_settings_arg = nullptr) {
     transaction::TransactionContext *txn = nullptr;
     std::unique_ptr<catalog::CatalogAccessor> accessor = nullptr;
-    std::vector<std::vector<parser::ConstantValueExpression>> param_ref = *params;
+    const auto &params_ref = *params;
 
     execution::exec::NoOpResultConsumer consumer;
     execution::exec::OutputCallback callback = consumer;
@@ -553,14 +571,23 @@ class ExecutionRunners : public benchmark::Fixture {
         exec_settings = *exec_settings_arg;
       }
 
-      auto exec_ctx = std::make_unique<execution::exec::ExecutionContext>(
-          db_oid, common::ManagedPointer(txn), callback, out_schema, common::ManagedPointer(accessor), exec_settings,
-          metrics_manager, DISABLED, DISABLED);
-
-      // Attach params to ExecutionContext
-      if (static_cast<size_t>(i) < param_ref.size()) {
-        exec_ctx->SetParams(common::ManagedPointer<const std::vector<parser::ConstantValueExpression>>(&param_ref[i]));
+      // TODO(Kyle): This makes an unnecessary copy of the query parameters
+      std::vector<parser::ConstantValueExpression> parameters{};
+      if (static_cast<std::size_t>(i) < params_ref.size()) {
+        std::copy(params_ref[i].cbegin(), params_ref[i].cend(), std::back_inserter(parameters));
       }
+      auto exec_ctx = execution::exec::ExecutionContextBuilder()
+                          .WithDatabaseOID(db_oid)
+                          .WithQueryParametersFrom(parameters)
+                          .WithExecutionSettings(exec_settings)
+                          .WithTxnContext(common::ManagedPointer{txn})
+                          .WithOutputSchema(common::ManagedPointer{out_schema})
+                          .WithOutputCallback(callback)
+                          .WithCatalogAccessor(common::ManagedPointer{accessor})
+                          .WithMetricsManager(metrics_manager_)
+                          .WithReplicationManager(DISABLED)
+                          .WithRecoveryManager(DISABLED)
+                          .Build();
 
       exec_query->Run(common::ManagedPointer(exec_ctx), mode);
 
@@ -582,10 +609,17 @@ class ExecutionRunners : public benchmark::Fixture {
     auto txn = txn_manager_->BeginTransaction();
     auto accessor = catalog_->GetAccessor(common::ManagedPointer(txn), db_oid, DISABLED);
     auto exec_settings = GetExecutionSettings();
-    auto exec_ctx = std::make_unique<execution::exec::ExecutionContext>(
-        db_oid, common::ManagedPointer(txn), nullptr, nullptr, common::ManagedPointer(accessor), exec_settings,
-        metrics_manager_, DISABLED, DISABLED);
-    exec_ctx->SetExecutionMode(static_cast<uint8_t>(mode));
+    auto exec_ctx = execution::exec::ExecutionContextBuilder()
+                        .WithDatabaseOID(db_oid)
+                        .WithExecutionSettings(exec_settings)
+                        .WithTxnContext(common::ManagedPointer{txn})
+                        .WithOutputSchema(execution::exec::ExecutionContext::NULL_OUTPUT_SCHEMA)
+                        .WithOutputCallback(execution::exec::ExecutionContext::NULL_OUTPUT_CALLBACK)
+                        .WithCatalogAccessor(common::ManagedPointer{accessor})
+                        .WithMetricsManager(metrics_manager_)
+                        .WithReplicationManager(DISABLED)
+                        .WithRecoveryManager(DISABLED)
+                        .Build();
 
     selfdriving::PipelineOperatingUnits units;
     selfdriving::ExecutionOperatingUnitFeatureVector pipe0_vec;
@@ -939,15 +973,24 @@ BENCHMARK_DEFINE_F(ExecutionRunners, SEQ0_OutputRunners)(benchmark::State &state
 
   auto txn = txn_manager_->BeginTransaction();
   auto accessor = catalog_->GetAccessor(common::ManagedPointer(txn), db_oid, DISABLED);
-  auto schema = std::make_unique<planner::OutputSchema>(std::move(cols));
+  auto schema = std::make_unique<const planner::OutputSchema>(std::move(cols));
 
   auto exec_settings = GetExecutionSettings();
   execution::compiler::ExecutableQuery::query_identifier.store(ExecutionRunners::query_id++);
   execution::exec::NoOpResultConsumer consumer;
   execution::exec::OutputCallback callback = consumer;
-  auto exec_ctx = std::make_unique<execution::exec::ExecutionContext>(
-      db_oid, common::ManagedPointer(txn), callback, schema.get(), common::ManagedPointer(accessor), exec_settings,
-      metrics_manager_, DISABLED, DISABLED);
+
+  auto exec_ctx = execution::exec::ExecutionContextBuilder()
+                      .WithDatabaseOID(db_oid)
+                      .WithExecutionSettings(exec_settings)
+                      .WithTxnContext(common::ManagedPointer{txn})
+                      .WithOutputSchema(common::ManagedPointer{schema})
+                      .WithOutputCallback(callback)
+                      .WithCatalogAccessor(common::ManagedPointer{accessor})
+                      .WithMetricsManager(metrics_manager_)
+                      .WithReplicationManager(DISABLED)
+                      .WithRecoveryManager(DISABLED)
+                      .Build();
 
   auto exec_query = execution::compiler::ExecutableQuery(output.str(), common::ManagedPointer(exec_ctx), false, 16,
                                                          exec_settings, txn->StartTime());
@@ -1011,9 +1054,18 @@ void ExecutionRunners::ExecuteIndexOperation(benchmark::State *state, bool is_in
     auto exec_settings = GetExecutionSettings();
     execution::exec::NoOpResultConsumer consumer;
     execution::exec::OutputCallback callback = consumer;
-    auto exec_ctx = std::make_unique<execution::exec::ExecutionContext>(
-        db_oid, common::ManagedPointer(txn), callback, nullptr, common::ManagedPointer(accessor), exec_settings,
-        metrics_manager, DISABLED, DISABLED);
+
+    auto exec_ctx = execution::exec::ExecutionContextBuilder()
+                        .WithDatabaseOID(db_oid)
+                        .WithExecutionSettings(exec_settings)
+                        .WithTxnContext(common::ManagedPointer{txn})
+                        .WithOutputSchema(execution::exec::ExecutionContext::NULL_OUTPUT_SCHEMA)
+                        .WithOutputCallback(callback)
+                        .WithCatalogAccessor(common::ManagedPointer{accessor})
+                        .WithMetricsManager(metrics_manager_)
+                        .WithReplicationManager(DISABLED)
+                        .WithRecoveryManager(DISABLED)
+                        .Build();
 
     // A brief discussion of the features:
     // NUM_ROWS: size of the index
@@ -2062,9 +2114,17 @@ void InitializeRunnersState() {
   // Load the database
   auto accessor = catalog->GetAccessor(common::ManagedPointer(txn), db_oid, DISABLED);
   auto exec_settings = ExecutionRunners::GetExecutionSettings();
-  auto exec_ctx = std::make_unique<execution::exec::ExecutionContext>(
-      db_oid, common::ManagedPointer(txn), nullptr, nullptr, common::ManagedPointer(accessor), exec_settings,
-      db_main->GetMetricsManager(), DISABLED, DISABLED);
+  auto exec_ctx = execution::exec::ExecutionContextBuilder()
+                      .WithDatabaseOID(db_oid)
+                      .WithExecutionSettings(exec_settings)
+                      .WithTxnContext(common::ManagedPointer{txn})
+                      .WithOutputSchema(execution::exec::ExecutionContext::NULL_OUTPUT_SCHEMA)
+                      .WithOutputCallback(execution::exec::ExecutionContext::NULL_OUTPUT_CALLBACK)
+                      .WithCatalogAccessor(common::ManagedPointer{accessor})
+                      .WithMetricsManager(db_main->GetMetricsManager())
+                      .WithReplicationManager(DISABLED)
+                      .WithRecoveryManager(DISABLED)
+                      .Build();
 
   execution::sql::TableGenerator table_gen(exec_ctx.get(), block_store, accessor->GetDefaultNamespace());
   table_gen.GenerateExecutionRunnersData(settings, config);