Setup tox

.circleci/config.yml

@@ -34,9 +34,11 @@ jobs:
             POSTGRES_TEST_PASS: ""
             POSTGRES_TEST_PORT: 5432
             POSTGRES_TEST_DBNAME: circle_test
+            POSTGRES_TEST_SCHEMA: codegen_integration_tests_postgres
           command: |
             . dbt_venv/bin/activate
             cd integration_tests
+            cp ci/sample.profiles.yml profiles.yml
             dbt --warn-error deps --target postgres
             dbt --warn-error run-operation create_source_table --target postgres
             dbt --warn-error seed --target postgres --full-refresh
@@ -45,10 +47,13 @@ jobs:
 
       - run:
           name: "Run Tests - Redshift"
+          environment:
+            REDSHIFT_TEST_SCHEMA: codegen_integration_tests_redshift
           command: |
             . dbt_venv/bin/activate
             echo `pwd`
             cd integration_tests
+            cp ci/sample.profiles.yml profiles.yml
             dbt --warn-error deps --target redshift
             dbt --warn-error run-operation create_source_table --target redshift
             dbt --warn-error seed --target redshift --full-refresh
@@ -57,10 +62,13 @@ jobs:
 
       - run:
           name: "Run Tests - Snowflake"
+          environment:
+            SNOWFLAKE_TEST_SCHEMA: codegen_integration_tests_snowflake
           command: |
             . dbt_venv/bin/activate
             echo `pwd`
             cd integration_tests
+            cp ci/sample.profiles.yml profiles.yml
             dbt --warn-error deps --target snowflake
             dbt --warn-error run-operation create_source_table --target snowflake
             dbt --warn-error seed --target snowflake --full-refresh
@@ -71,11 +79,13 @@ jobs:
           name: "Run Tests - BigQuery"
           environment:
             BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
+            REDSHIFT_TEST_SCHEMA: codegen_integration_tests_bigquery
 
           command: |
             . dbt_venv/bin/activate
             echo `pwd`
             cd integration_tests
+            cp ci/sample.profiles.yml profiles.yml
             dbt --warn-error deps --target bigquery
             dbt --warn-error run-operation create_source_table --target bigquery
             dbt --warn-error seed --target bigquery --full-refresh

.github/workflows/ci.yml

@@ -0,0 +1,139 @@
+# **what?**
+# Run tests for dbt-codegen against supported adapters
+
+# **why?**
+# To ensure that dbt-codegen works as expected with all supported adapters
+
+# **when?**
+# On every PR, and every push to main and when manually triggered
+
+name: Package Integration Tests
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+    workflow_dispatch:
+        inputs:
+            adapter:
+                description: The adapter to test against.  Defaults to all supported adapters when blank.
+                type: string
+                required: false
+
+env:
+    PYTHON_VERSION: "3.11"
+
+jobs:
+    determine-supported-adapters:
+        runs-on: ubuntu-latest
+        outputs:
+            adapters: ${{ steps.supported-adapters.outputs.adapters }}
+        steps:
+            - name: "Checkout ${{ github.event.repository }}"
+              uses: actions/checkout@v4
+
+            - name: "Set up Python ${{ env.PYTHON_VERSION }}"
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ env.PYTHON_VERSION }}
+
+            - name: "Install tox"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install tox
+
+            - name: "Get list of supported adapters or use input adapter only"
+              id: list-adapters
+              run: |
+                  if [ -z "${{ inputs.adapter }}" ]; then
+                      # github adds a pip freeze and a new line we need to strip out
+                      source supported_adapters.env
+                      echo $SUPPORTED_ADAPTERS
+                      echo "test_adapters=$SUPPORTED_ADAPTERS" >> $GITHUB_OUTPUT
+                  else
+                      echo "test_adapters=${{ inputs.adapter }}" >> $GITHUB_OUTPUT
+                  fi
+
+            - name: "Format adapter list for use as the matrix"
+              id: supported-adapters
+              run: |
+                  # Convert to JSON array and output
+                  supported_adapters=$(echo "${{ steps.list-adapters.outputs.test_adapters }}" | jq -Rc 'split(",")')
+                  echo $supported_adapters
+                  echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT
+
+            - name: "[ANNOTATION] ${{ github.event.repository.name }} - Testing ${{ steps.supported-adapters.outputs.adapters }}"
+              run: |
+                  title="${{ github.event.repository.name }} - adapters to test"
+                  message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}"
+                  echo "::notice $title::$message"
+
+    run-tests:
+        runs-on: ubuntu-latest
+        needs: [determine-supported-adapters]
+        services:
+            postgres:
+                image: postgres
+                env:
+                    POSTGRES_USER: ${{ vars.POSTGRES_USER }}
+                    POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASS }}
+                    POSTGRES_DB: ${{ vars.POSTGRES_DATABASE }}
+                    POSTGRES_HOST: ${{ vars.POSTGRES_HOST }}
+                options: >-
+                    --health-cmd pg_isready
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+                ports:
+                    - 5432:5432
+        strategy:
+            fail-fast: false
+            matrix:
+                adapter: ${{fromJson(needs.determine-supported-adapters.outputs.adapters)}}
+
+        steps:
+            - name: "Checkout ${{ github.event.repository }} "
+              uses: actions/checkout@v4
+
+            - name: "Set up Python ${{ env.PYTHON_VERSION }}"
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ env.PYTHON_VERSION }}
+
+            - name: "Install ${{ matrix.adapter }}"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install dbt-${{ matrix.adapter }}
+
+            - name: "Install tox"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install tox
+
+            - name: "Run integration tests with tox on ${{ matrix.adapter }}"
+              run: |
+                  tox -e dbt_integration_${{ matrix.adapter }}
+              env:
+                  POSTGRES_HOST: ${{ vars.POSTGRES_HOST }}
+                  POSTGRES_USER: ${{ vars.POSTGRES_USER }}
+                  DBT_ENV_SECRET_POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }}
+                  POSTGRES_PORT: 5432
+                  POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }}
+                  POSTGRES_SCHEMA: "dbt_codegen_integration_tests_postgres_${{ github.run_number }}"
+                  SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+                  SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
+                  DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
+                  SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
+                  SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
+                  SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
+                  SNOWFLAKE_SCHEMA: "dbt_codegen_integration_tests_snowflake_${{ github.run_number }}"
+                  REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
+                  REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
+                  DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
+                  REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
+                  REDSHIFT_SCHEMA: "dbt_codegen_integration_tests_redshift_${{ github.run_number }}"
+                  REDSHIFT_PORT: 5439
+                  BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
+                  BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
+                  BIGQUERY_SCHEMA: "dbt_codegen_integration_tests_bigquery_${{ github.run_number }}"

integration_tests/README.md

@@ -48,30 +48,30 @@ The environment variables you'll need to set for each adapter are:
 
 ```bash
 # Postgres — these are the defaults for the Docker container so actually have values
-export POSTGRES_TEST_HOST=localhost
-export POSTGRES_TEST_USER=root
-export POSTGRES_TEST_PASS=''
-export POSTGRES_TEST_PORT=5432
-export POSTGRES_TEST_DBNAME=circle_test
+export POSTGRES_HOST=localhost
+export POSTGRES_USER=root
+export POSTGRES_PASS=''
+export POSTGRES_PORT=5432
+export POSTGRES_DATABASE=circle_test
 
 # BigQuery
 export BIGQUERY_SERVICE_KEY_PATH=
-export BIGQUERY_TEST_DATABASE=
+export BIGQUERY_PROJECT=
 
 # Redshift
-export REDSHIFT_TEST_HOST=
-export REDSHIFT_TEST_USER=
-export REDSHIFT_TEST_PASS=
-export REDSHIFT_TEST_DBNAME=
-export REDSHIFT_TEST_PORT=
+export REDSHIFT_HOST=
+export REDSHIFT_USER=
+export REDSHIFT_PASS=
+export REDSHIFT_DATABASE=
+export REDSHIFT_PORT=
 
 # Snowflake
-export SNOWFLAKE_TEST_ACCOUNT=
-export SNOWFLAKE_TEST_USER=
-export SNOWFLAKE_TEST_PASSWORD=
-export SNOWFLAKE_TEST_ROLE=
-export SNOWFLAKE_TEST_DATABASE=
-export SNOWFLAKE_TEST_WAREHOUSE=
+export SNOWFLAKE_ACCOUNT=
+export SNOWFLAKE_USER=
+export SNOWFLAKE_PASSWORD=
+export SNOWFLAKE_ROLE=
+export SNOWFLAKE_DATABASE=
+export SNOWFLAKE_WAREHOUSE=
 ```
 
 ### Setup Postgres or other database targets

integration_tests/ci/sample.profiles.yml

@@ -0,0 +1,48 @@
+
+# HEY! This file is used in the dbt-codegen integrations tests with CircleCI.
+# You should __NEVER__ check credentials into version control. Thanks for reading :)
+
+# This profile only exists for use with circleCI.
+# Once CircleCi is deprecated, this profile can be removed.
+
+integration_tests:
+  target: postgres
+  outputs:
+    postgres:
+      type: postgres
+      host: "{{ env_var('POSTGRES_TEST_HOST') }}"
+      user: "{{ env_var('POSTGRES_TEST_USER') }}"
+      pass: "{{ env_var('POSTGRES_TEST_PASS') }}"
+      port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}"
+      schema: codegen_integration_tests_postgres
+      threads: 1
+
+    redshift:
+      type: redshift
+      host: "{{ env_var('REDSHIFT_TEST_HOST') }}"
+      user: "{{ env_var('REDSHIFT_TEST_USER') }}"
+      pass: "{{ env_var('REDSHIFT_TEST_PASS') }}"
+      dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
+      port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
+      schema: codegen_integration_tests_redshift
+      threads: 1
+
+    bigquery:
+      type: bigquery
+      method: service-account
+      keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
+      project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
+      schema: codegen_integration_tests_bigquery
+      threads: 1
+
+    snowflake:
+      type: snowflake
+      account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}"
+      user: "{{ env_var('SNOWFLAKE_TEST_USER') }}"
+      password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}"
+      role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}"
+      database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
+      warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
+      schema: codegen_integration_tests_snowflake
+      threads: 1

integration_tests/dbt_project.yml

@@ -15,6 +15,10 @@ clean-targets:
   - "target"
   - "dbt_packages"
 
+flags:
+  send_anonymous_usage_stats: False
+  use_colors: True
+
 seeds:
   +schema: raw_data
   +quote_columns: false

integration_tests/profiles.yml

@@ -1,49 +1,48 @@
 
-# HEY! This file is used in the dbt-codegen integrations tests with CircleCI.
-# You should __NEVER__ check credentials into version control. Thanks for reading :)
-
-config:
-    send_anonymous_usage_stats: False
-    use_colors: True
+# HEY! This file is used in the dbt-utils integrations tests with CircleCI.
+# You should __NEVER__ check credentials into version control. That's why we use environment variables everywhere.
+# Thanks for reading :)
 
 integration_tests:
   target: postgres
   outputs:
     postgres:
-      type: postgres
-      host: "{{ env_var('POSTGRES_TEST_HOST') }}"
-      user: "{{ env_var('POSTGRES_TEST_USER') }}"
-      pass: "{{ env_var('POSTGRES_TEST_PASS') }}"
-      port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
-      dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}"
-      schema: codegen_integration_tests_postgres
-      threads: 1
+      type: "postgres"
+      host: "{{ env_var('POSTGRES_HOST') }}"
+      user: "{{ env_var('POSTGRES_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}"
+      port: "{{ env_var('POSTGRES_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_DATABASE') }}"
+      schema: "{{ env_var('POSTGRES_SCHEMA') }}"
+      threads: 5
 
     redshift:
-      type: redshift
-      host: "{{ env_var('REDSHIFT_TEST_HOST') }}"
-      user: "{{ env_var('REDSHIFT_TEST_USER') }}"
-      pass: "{{ env_var('REDSHIFT_TEST_PASS') }}"
-      dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
-      port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
-      schema: codegen_integration_tests_redshift
-      threads: 1
+      type: "redshift"
+      host: "{{ env_var('REDSHIFT_HOST') }}"
+      user: "{{ env_var('REDSHIFT_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}"
+      dbname: "{{ env_var('REDSHIFT_DATABASE') }}"
+      port: "{{ env_var('REDSHIFT_PORT') | as_number }}"
+      schema: "{{ env_var('REDSHIFT_SCHEMA') }}"
+      threads: 5
 
     bigquery:
-      type: bigquery
-      method: service-account
-      keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
-      project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
-      schema: codegen_integration_tests_bigquery
-      threads: 1
+      type: "bigquery"
+      method: "service-account-json"
+      project: "{{ env_var('BIGQUERY_PROJECT') }}"
+      dataset: "{{ env_var('BIGQUERY_SCHEMA') }}"
+      threads: 10
+      keyfile_json:
+        "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}"
+      job_retries: 3
 
     snowflake:
-      type: snowflake
-      account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}"
-      user: "{{ env_var('SNOWFLAKE_TEST_USER') }}"
-      password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}"
-      role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}"
-      database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
-      warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
-      schema: codegen_integration_tests_snowflake
-      threads: 1
+      type: "snowflake"
+      account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
+      user: "{{ env_var('SNOWFLAKE_USER') }}"
+      password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}"
+      role: "{{ env_var('SNOWFLAKE_ROLE') }}"
+      database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
+      warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"
+      schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}"
+      threads: 10

supported_adapters.env

@@ -0,0 +1 @@
+SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery