Merge branch 'main' into update/create-drop-schemas

.github/scripts/integration-test-matrix.js

@@ -1,6 +1,6 @@
 module.exports = ({ context }) => {
   const defaultPythonVersion = "3.8";
-  const supportedPythonVersions = ["3.7", "3.8", "3.9"];
+  const supportedPythonVersions = ["3.7", "3.8", "3.9", "3.10"];
   const supportedAdapters = ["bigquery"];
 
   // if PR, generate matrix based on files changed and PR labels

.github/workflows/integration.yml

@@ -153,14 +153,14 @@ jobs:
 
       - name: Install python dependencies
         run: |
-          pip install --user --upgrade pip
-          pip install tox
-          pip --version
+          python -m pip install --user --upgrade pip
+          python -m pip install tox
+          python -m pip --version
           tox --version
 
       - name: Install dbt-core latest
         run: |
-          pip install "git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core"
+          python -m pip install "git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core"
 
       - name: Run tox (bigquery)
         if: matrix.adapter == 'bigquery'

.github/workflows/main.yml

@@ -52,9 +52,9 @@ jobs:
 
       - name: Install python dependencies
         run: |
-          pip install --user --upgrade pip
-          pip install -r dev_requirements.txt
-          pip --version
+          python -m pip install --user --upgrade pip
+          python -m pip install -r dev-requirements.txt
+          python -m pip --version
           pre-commit --version
           mypy --version
           dbt --version
@@ -69,7 +69,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8] # TODO: support unit testing for python 3.9 (https://github.com/dbt-labs/dbt/issues/3689)
+        python-version: ['3.7', '3.8', '3.9', '3.10']
 
     env:
       TOXENV: "unit"
@@ -88,13 +88,13 @@ jobs:
 
       - name: Install python dependencies
         run: |
-          pip install --user --upgrade pip
-          pip install tox
-          pip --version
+          python -m pip install --user --upgrade pip
+          python -m pip install tox
+          python -m pip --version
           tox --version
       - name: Install dbt-core latest
         run: |
-          pip install "git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core"
+          python -m pip install "git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core"
       - name: Run tox
         run: tox
 
@@ -130,9 +130,9 @@ jobs:
 
       - name: Install python dependencies
         run: |
-          pip install --user --upgrade pip
-          pip install --upgrade setuptools wheel twine check-wheel-contents
-          pip --version
+          python -m pip install --user --upgrade pip
+          python -m pip install --upgrade setuptools wheel twine check-wheel-contents
+          python -m pip --version
 
       - name: Build distributions
         run: ./scripts/build-dist.sh
@@ -173,7 +173,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.7, 3.8, 3.9]
+        python-version: ['3.7', '3.8', '3.9', '3.10']
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
@@ -183,9 +183,9 @@ jobs:
 
       - name: Install python dependencies
         run: |
-          pip install --user --upgrade pip
-          pip install --upgrade wheel
-          pip --version
+          python -m pip install --user --upgrade pip
+          python -m pip install --upgrade wheel
+          python -m pip --version
       - uses: actions/download-artifact@v2
         with:
           name: dist
@@ -196,13 +196,13 @@ jobs:
 
       - name: Install wheel distributions
         run: |
-          find ./dist/*.whl -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
+          find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
       - name: Check wheel distributions
         run: |
           dbt --version
       - name: Install source distributions
         run: |
-          find ./dist/*.gz -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
+          find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
       - name: Check source distributions
         run: |
           dbt --version

.github/workflows/version-bump.yml

@@ -57,7 +57,7 @@ jobs:
         run: |
           python3 -m venv env
           source env/bin/activate
-          pip install --upgrade pip
+          python -m pip install --upgrade pip
 
       - name: Create PR branch
         if: ${{ steps.variables.outputs.IS_DRY_RUN  == 'true' }}
@@ -69,7 +69,7 @@ jobs:
       - name: Bumping version
         run: |
           source env/bin/activate
-          pip install -r dev_requirements.txt
+          python -m pip install -r dev-requirements.txt
           env/bin/bumpversion --allow-dirty --new-version ${{steps.variables.outputs.VERSION_NUMBER}} major
           git status
 

.pre-commit-config.yaml

@@ -39,7 +39,7 @@ repos:
     alias: flake8-check
     stages: [manual]
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.782
+  rev: v0.942
   hooks:
   - id: mypy
     args: [--show-error-codes, --ignore-missing-imports]

CHANGELOG.md

@@ -1,4 +1,9 @@
-## dbt-bigquery 1.1.0 (Release TBD)
+## dbt-bigquery 1.2.0 (Release TBD)
+- Adding Python 3.10 testing and enabling mypy ([#177](https://github.com/dbt-labs/dbt-bigquery/pull/177))
+
+## dbt-bigquery 1.1.0 (April 28, 2022)
+
+## dbt-bigquery 1.1.0rc2 (April 20, 2022)
 
 ### Fixes
 - Restore default behavior for query timeout. Set default `job_execution_timeout` to `None` by default. Keep 300 seconds as query timeout where previously used.

dbt/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

dbt/adapters/bigquery/connections.py

@@ -555,6 +555,9 @@ def table_ref(database, schema, table_name):
     def get_bq_table(self, database, schema, identifier):
         """Get a bigquery table for a schema/model."""
         conn = self.get_thread_connection()
+        # backwards compatibility: fill in with defaults if not specified
+        database = database or conn.credentials.database
+        schema = schema or conn.credentials.schema
         table_ref = self.table_ref(database, schema, identifier)
         return conn.handle.get_table(table_ref)
 

dbt/adapters/bigquery/impl.py

@@ -318,7 +318,7 @@ def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str:
 
     @classmethod
     def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
-        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
+        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))  # type: ignore[attr-defined]
         return "float64" if decimals else "int64"
 
     @classmethod

dev_requirements.txt → dev-requirements.txt

@@ -9,7 +9,7 @@ flake8
 flaky
 freezegun==1.1.0
 ipdb
-mypy==0.782
+mypy==0.942
 pip-tools
 pre-commit
 pytest
@@ -19,5 +19,6 @@ pytest-csv
 pytest-xdist
 pytz
 tox>=3.13
+types-requests
 twine
 wheel

mypy.ini

@@ -1,3 +1,4 @@
 [mypy]
-mypy_path = ./third-party-stubs
+mypy_path = third-party-stubs/
 namespace_packages = True
+exclude = third-party-stubs/*

setup.py

@@ -83,6 +83,7 @@ def _get_dbt_core_version():
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
     python_requires=">=3.7",
 )

test.env.example

@@ -1,4 +1,3 @@
-BIGQUERY_TEST_DATABASE=
 BIGQUERY_TEST_ALT_DATABASE=
 BIGQUERY_TEST_NO_ACCESS_DATABASE=
 BIGQUERY_TEST_SERVICE_ACCOUNT_JSON='{}'

tests/functional/adapter/expected_stats.py

@@ -0,0 +1,62 @@
+from dbt.tests.util import AnyString, AnyFloat
+
+
+def bigquery_stats(is_table, partition=None, cluster=None):
+    stats = {}
+
+    if is_table:
+        stats.update(
+            {
+                "num_bytes": {
+                    "id": "num_bytes",
+                    "label": AnyString(),
+                    "value": AnyFloat(),
+                    "description": AnyString(),
+                    "include": True,
+                },
+                "num_rows": {
+                    "id": "num_rows",
+                    "label": AnyString(),
+                    "value": AnyFloat(),
+                    "description": AnyString(),
+                    "include": True,
+                },
+            }
+        )
+
+    if partition is not None:
+        stats.update(
+            {
+                "partitioning_type": {
+                    "id": "partitioning_type",
+                    "label": AnyString(),
+                    "value": partition,
+                    "description": AnyString(),
+                    "include": True,
+                }
+            }
+        )
+
+    if cluster is not None:
+        stats.update(
+            {
+                "clustering_fields": {
+                    "id": "clustering_fields",
+                    "label": AnyString(),
+                    "value": cluster,
+                    "description": AnyString(),
+                    "include": True,
+                }
+            }
+        )
+
+    has_stats = {
+        "id": "has_stats",
+        "label": "Has Stats?",
+        "value": bool(stats),
+        "description": "Indicates whether there are statistics for this table",
+        "include": False,
+    }
+    stats["has_stats"] = has_stats
+
+    return stats

tests/functional/adapter/test_basic.py

@@ -13,6 +13,9 @@
 from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp
 from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod
 from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection
+from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate
+from dbt.tests.adapter.basic.expected_catalog import base_expected_catalog
+from tests.functional.adapter.expected_stats import bigquery_stats
 
 
 class TestSimpleMaterializationsBigQuery(BaseSimpleMaterializations):
@@ -53,8 +56,26 @@ class TestSnapshotCheckColsBigQuery(BaseSnapshotCheckCols):
 class TestSnapshotTimestampBigQuery(BaseSnapshotTimestamp):
     pass
 
+
 class TestBaseAdapterMethodBigQuery(BaseAdapterMethod):
     pass
 
+
 class TestBigQueryValidateConnection(BaseValidateConnection):
-    pass
+    pass
+
+
+class TestDocsGenerateBigQuery(BaseDocsGenerate):
+    @pytest.fixture(scope="class")
+    def expected_catalog(self, project):
+        return base_expected_catalog(
+            project,
+            role=None,
+            id_type="INT64",
+            text_type="STRING",
+            time_type="DATETIME",
+            view_type="view",
+            table_type="table",
+            model_stats=bigquery_stats(False),
+            seed_stats=bigquery_stats(True),
+        )

tests/functional/test_get_columns_incomplete_database.py

@@ -0,0 +1,54 @@
+import pytest
+from dbt.tests.util import run_dbt
+
+# This is to test a edge case found in https://github.com/dbt-labs/dbt-bigquery/pull/165/files
+
+tests__get_cols_in_sql = """
+{% test get_cols_in(model) %}
+
+  {# The step which causes the issue #}
+  {%- set relation = api.Relation.create(identifier=model.table) if execute -%}
+
+  {% set columns = adapter.get_columns_in_relation(relation) %}
+
+  select
+    {% for col in columns %}
+      {{ col.name }} {{ "," if not loop.last }}
+    {% endfor %}
+
+  from {{ model }}
+  limit 0
+
+{% endtest %}
+"""
+
+models__my_model = """select 1 as id, 'text' as another_col
+"""
+
+properties__model_yml = """
+version: 2
+models:
+  - name: my_model
+    tests:
+      - get_cols_in
+"""
+
+class TestIncompleteRelationSetup:
+    @pytest.fixture(scope="class")
+    def properties(self):
+        return {"properties__model_yml.yml": properties__model_yml}
+
+    @pytest.fixture(scope="class")
+    def macros(self):
+        return {"get_col_in.sql": tests__get_cols_in_sql}
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return { "my_model.sql": models__my_model }
+
+
+class TestIncompleteRelation(TestIncompleteRelationSetup):
+
+  def test_incomplete_relation(self, project):
+     run_dbt(["build"])
+