From 15f35b4ff7658fff0783b40fb7b0347344692f4c Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Thu, 5 May 2022 13:05:03 -0500 Subject: [PATCH 1/6] fixing filename in contributing.md (#149) * fixing filename in contributing.md * rename dev_requirements.txt -> dev-requirements.txt to match dbt-core * catch all refs to dev_requirements for rename --- .github/workflows/main.yml | 2 +- .github/workflows/version-bump.yml | 2 +- dev_requirements.txt => dev-requirements.txt | 0 tox.ini | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename dev_requirements.txt => dev-requirements.txt (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bc30363c1..d2ed07654 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,7 +53,7 @@ jobs: - name: Install python dependencies run: | pip install --user --upgrade pip - pip install -r dev_requirements.txt + pip install -r dev-requirements.txt pip --version pre-commit --version mypy --version diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index b0a3174df..dbd36786a 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -69,7 +69,7 @@ jobs: - name: Bumping version run: | source env/bin/activate - pip install -r dev_requirements.txt + pip install -r dev-requirements.txt env/bin/bumpversion --allow-dirty --new-version ${{steps.variables.outputs.VERSION_NUMBER}} major git status diff --git a/dev_requirements.txt b/dev-requirements.txt similarity index 100% rename from dev_requirements.txt rename to dev-requirements.txt diff --git a/tox.ini b/tox.ini index 8ddd4795c..a5da0d0a6 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ skip_install = true passenv = DBT_* PYTEST_ADDOPTS commands = {envpython} -m pytest {posargs} tests/unit deps = - -rdev_requirements.txt + -rdev-requirements.txt -e. [testenv:{integration,py37,py38,py39,py}-{bigquery}] @@ -19,5 +19,5 @@ commands = bigquery: {envpython} -m pytest {posargs} -m profile_bigquery tests/integration bigquery: {envpython} -m pytest {posargs} tests/functional --profile service_account deps = - -rdev_requirements.txt + -rdev-requirements.txt -e. From 8aaa6ab6da182aa8e831f631feb12af5ec76d9ce Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Wed, 11 May 2022 11:20:26 -0500 Subject: [PATCH 2/6] init push or pytest conversion of override_database for bigquery (#165) * init push or pytest conversion of override_database for bigquery * tests are currently passing, need to implement some test changes overs (commented out tests) * implemented a vew versions of the check_realations_equal test on test_bigquery_database_override test working on getting the rest to work for first test * trying to grab local env to test against need to find way to grab information from configs * attempting to move over to using check_relations_equal_with_relations * ordering of models * post pair update * debugged test, had passing locally * adding change to conftest to see if it passes tests in ci/cd * removing conftest change * trying ci/cd after some changes * trying to take into account CT-604, remove old test, remove BIGQUERY_TEST_DATABASE env * moving override_database out of adapter folder up one --- dbt/adapters/bigquery/connections.py | 3 + test.env.example | 1 - .../test_override_database/fixtures.py | 71 +++++++ .../test_override_database.py | 158 ++++++++++++++++ .../models/subfolder/view_3.sql | 1 - .../models/subfolder/view_4.sql | 5 - .../override_database_test/models/view_1.sql | 7 - .../override_database_test/models/view_2.sql | 6 - .../override_database_test/seeds/seed.csv | 6 - .../test_override_database.py | 176 ------------------ 10 files changed, 232 insertions(+), 202 deletions(-) create mode 100644 tests/functional/test_override_database/fixtures.py create mode 100644 tests/functional/test_override_database/test_override_database.py delete mode 100644 tests/integration/override_database_test/models/subfolder/view_3.sql delete mode 100644 tests/integration/override_database_test/models/subfolder/view_4.sql delete mode 100644 tests/integration/override_database_test/models/view_1.sql delete mode 100644 tests/integration/override_database_test/models/view_2.sql delete mode 100644 tests/integration/override_database_test/seeds/seed.csv delete mode 100644 tests/integration/override_database_test/test_override_database.py diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index a445796b5..454e84d7d 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -555,6 +555,9 @@ def table_ref(database, schema, table_name): def get_bq_table(self, database, schema, identifier): """Get a bigquery table for a schema/model.""" conn = self.get_thread_connection() + # backwards compatibility: fill in with defaults if not specified + database = database or conn.credentials.database + schema = schema or conn.credentials.schema table_ref = self.table_ref(database, schema, identifier) return conn.handle.get_table(table_ref) diff --git a/test.env.example b/test.env.example index 031968c60..2065e4393 100644 --- a/test.env.example +++ b/test.env.example @@ -1,4 +1,3 @@ -BIGQUERY_TEST_DATABASE= BIGQUERY_TEST_ALT_DATABASE= BIGQUERY_TEST_NO_ACCESS_DATABASE= BIGQUERY_TEST_SERVICE_ACCOUNT_JSON='{}' diff --git a/tests/functional/test_override_database/fixtures.py b/tests/functional/test_override_database/fixtures.py new file mode 100644 index 000000000..315a74e8f --- /dev/null +++ b/tests/functional/test_override_database/fixtures.py @@ -0,0 +1,71 @@ +import pytest +from dbt.tests.fixtures.project import write_project_files + + +models__view_2_sql = """ +{%- if target.type == 'bigquery' -%} + {{ config(project=var('alternate_db')) }} +{%- else -%} + {{ config(database=var('alternate_db')) }} +{%- endif -%} +select * from {{ ref('seed') }} + +""" + +models__view_1_sql = """ +{# + We are running against a database that must be quoted. + These calls ensure that we trigger an error if we're failing to quote at parse-time +#} +{% do adapter.already_exists(this.schema, this.table) %} +{% do adapter.get_relation(this.database, this.schema, this.table) %} +select * from {{ ref('seed') }} + +""" + +models__subfolder__view_4_sql = """ +{{ + config(database=var('alternate_db')) +}} + +select * from {{ ref('seed') }} + +""" + +models__subfolder__view_3_sql = """ +select * from {{ ref('seed') }} + +""" + +seeds__seed_csv = """id,name +1,a +2,b +3,c +4,d +5,e +""" + +@pytest.fixture(scope="class") +def models(): + return { + "view_2.sql": models__view_2_sql, + "view_1.sql": models__view_1_sql, + "subfolder": + { + "view_4.sql": models__subfolder__view_4_sql, + "view_3.sql": models__subfolder__view_3_sql, + }, + } + +@pytest.fixture(scope="class") +def seeds(): + return { + "seed.csv": seeds__seed_csv + } + +@pytest.fixture(scope="class") +def project_files(project_root, models, seeds,): + write_project_files(project_root, "models", models) + write_project_files(project_root, "seeds", seeds) + + diff --git a/tests/functional/test_override_database/test_override_database.py b/tests/functional/test_override_database/test_override_database.py new file mode 100644 index 000000000..8ce179056 --- /dev/null +++ b/tests/functional/test_override_database/test_override_database.py @@ -0,0 +1,158 @@ +import pytest +from dbt.tests.util import run_dbt, check_relations_equal, check_relations_equal_with_relations +from tests.functional.test_override_database.fixtures import ( + models, + seeds, + project_files +) +import os + + + + +class BaseOverrideDatabase: + @pytest.fixture(scope="class") + def model_path(self): + return "models" + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "seed-paths": ["seeds"], + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "quoting": { + "database": True, + }, + "seeds": { + "quote_columns": False, + } + } + + +class TestModelOverrideBigQuery(BaseOverrideDatabase): + def run_database_override(self, project): + run_dbt(["seed"]) + assert len(run_dbt(["run"])) == 4 + check_relations_equal_with_relations(project.adapter, [ + project.adapter.Relation.create(schema=project.test_schema, identifier="seed"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_2"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_1"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_3"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_4") + ]) + + + def test_bigquery_database_override(self, project): + self.run_database_override(project) + + +class BaseTestProjectModelOverrideBigQuery(BaseOverrideDatabase): + + def run_database_override(self, project): + run_dbt(["seed"]) + assert len(run_dbt(["run"])) == 4 + self.assertExpectedRelations(project) + + def assertExpectedRelations(self, project): + check_relations_equal_with_relations(project.adapter, [ + project.adapter.Relation.create(schema=project.test_schema, identifier="seed"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_2"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_1"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_3"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_4") + ]) + + +class TestProjectModelOverrideBigQuery(BaseTestProjectModelOverrideBigQuery): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "models": { + "database": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + "test": { + "subfolder": { + "database": "{{ target.database }}" + } + } + }, + "seed-paths": ["seeds"], + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "quoting": { + "database": True, + }, + "seeds": { + "quote_columns": False, + } + } + + def test_bigquery_database_override(self, project): + self.run_database_override(project) + + +class TestProjectModelAliasOverrideBigQuery(BaseTestProjectModelOverrideBigQuery): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "models": { + "project": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + "test": { + "subfolder": { + "project": "{{ target.database }}" + } + } + }, + "seed-paths": ["seeds"], + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "quoting": { + "database": True, + }, + "seeds": { + "quote_columns": False, + } + } + + def test_bigquery_project_override(self, project): + self.run_database_override(project) + + +class TestProjectSeedOverrideBigQuery(BaseOverrideDatabase): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "seed-paths": ["seeds"], + "vars": { + "alternate_db": os.getenv("BIGQUERY_TEST_ALT_DATABASE"), + }, + "seeds": { + "database": os.getenv("BIGQUERY_TEST_ALT_DATABASE") + } + } + def run_database_override(self, project): + run_dbt(["seed"]) + assert len(run_dbt(["run"])) == 4 + check_relations_equal_with_relations(project.adapter, [ + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="seed"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_2"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_1"), + project.adapter.Relation.create(schema=project.test_schema, identifier="view_3"), + project.adapter.Relation.create(database=os.getenv("BIGQUERY_TEST_ALT_DATABASE"), schema=project.test_schema, identifier="view_4") + ]) + + def test_bigquery_database_override(self, project): + self.run_database_override(project) diff --git a/tests/integration/override_database_test/models/subfolder/view_3.sql b/tests/integration/override_database_test/models/subfolder/view_3.sql deleted file mode 100644 index 4b91aa0f2..000000000 --- a/tests/integration/override_database_test/models/subfolder/view_3.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('seed') }} diff --git a/tests/integration/override_database_test/models/subfolder/view_4.sql b/tests/integration/override_database_test/models/subfolder/view_4.sql deleted file mode 100644 index efa1268fa..000000000 --- a/tests/integration/override_database_test/models/subfolder/view_4.sql +++ /dev/null @@ -1,5 +0,0 @@ -{{ - config(database=var('alternate_db')) -}} - -select * from {{ ref('seed') }} diff --git a/tests/integration/override_database_test/models/view_1.sql b/tests/integration/override_database_test/models/view_1.sql deleted file mode 100644 index a43f04646..000000000 --- a/tests/integration/override_database_test/models/view_1.sql +++ /dev/null @@ -1,7 +0,0 @@ -{# - We are running against a database that must be quoted. - These calls ensure that we trigger an error if we're failing to quote at parse-time -#} -{% do adapter.already_exists(this.schema, this.table) %} -{% do adapter.get_relation(this.database, this.schema, this.table) %} -select * from {{ ref('seed') }} diff --git a/tests/integration/override_database_test/models/view_2.sql b/tests/integration/override_database_test/models/view_2.sql deleted file mode 100644 index 9ac6bdad6..000000000 --- a/tests/integration/override_database_test/models/view_2.sql +++ /dev/null @@ -1,6 +0,0 @@ -{%- if target.type == 'bigquery' -%} - {{ config(project=var('alternate_db')) }} -{%- else -%} - {{ config(database=var('alternate_db')) }} -{%- endif -%} -select * from {{ ref('seed') }} diff --git a/tests/integration/override_database_test/seeds/seed.csv b/tests/integration/override_database_test/seeds/seed.csv deleted file mode 100644 index ae9125c16..000000000 --- a/tests/integration/override_database_test/seeds/seed.csv +++ /dev/null @@ -1,6 +0,0 @@ -id,name -1,a -2,b -3,c -4,d -5,e diff --git a/tests/integration/override_database_test/test_override_database.py b/tests/integration/override_database_test/test_override_database.py deleted file mode 100644 index 9b12b3e79..000000000 --- a/tests/integration/override_database_test/test_override_database.py +++ /dev/null @@ -1,176 +0,0 @@ -from tests.integration.base import DBTIntegrationTest, use_profile - -import os - - -class BaseOverrideDatabase(DBTIntegrationTest): - setup_alternate_db = True - @property - def schema(self): - return "override_database" - - @property - def models(self): - return "models" - - @property - def alternative_database(self): - return super().alternative_database - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'quoting': { - 'database': True, - }, - 'seeds': { - 'quote_columns': False, - } - } - - -class TestModelOverride(BaseOverrideDatabase): - def run_database_override(self): - func = lambda x: x - - self.run_dbt(['seed']) - - self.assertEqual(len(self.run_dbt(['run'])), 4) - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.default_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.default_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) - - @use_profile('bigquery') - def test_bigquery_database_override(self): - self.run_database_override() - - -class BaseTestProjectModelOverride(BaseOverrideDatabase): - # this is janky, but I really want to access self.default_database in - # project_config - @property - def default_database(self): - target = self._profile_config['test']['target'] - profile = self._profile_config['test']['outputs'][target] - for key in ['database', 'project', 'dbname']: - if key in profile: - database = profile[key] - return database - assert False, 'No profile database found!' - - def run_database_override(self): - self.run_dbt(['seed']) - self.assertEqual(len(self.run_dbt(['run'])), 4) - self.assertExpectedRelations() - - def assertExpectedRelations(self): - func = lambda x: x - - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.default_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.alternative_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) - - -class TestProjectModelOverride(BaseTestProjectModelOverride): - @property - def project_config(self): - return { - 'config-version': 2, - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'models': { - 'database': self.alternative_database, - 'test': { - 'subfolder': { - 'database': self.default_database, - } - } - }, - 'seed-paths': ['seeds'], - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'quoting': { - 'database': True, - }, - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile('bigquery') - def test_bigquery_database_override(self): - self.run_database_override() - - -class TestProjectModelAliasOverride(BaseTestProjectModelOverride): - @property - def project_config(self): - return { - 'config-version': 2, - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'models': { - 'project': self.alternative_database, - 'test': { - 'subfolder': { - 'project': self.default_database, - } - } - }, - 'seed-paths': ['seeds'], - 'vars': { - 'alternate_db': self.alternative_database, - }, - 'quoting': { - 'database': True, - }, - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile('bigquery') - def test_bigquery_project_override(self): - self.run_database_override() - - -class TestProjectSeedOverride(BaseOverrideDatabase): - def run_database_override(self): - func = lambda x: x - - self.use_default_project({ - 'config-version': 2, - 'seeds': { - 'database': self.alternative_database - }, - }) - self.run_dbt(['seed']) - - self.assertEqual(len(self.run_dbt(['run'])), 4) - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.alternative_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.default_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) - - @use_profile('bigquery') - def test_bigquery_database_override(self): - self.run_database_override() From 824052240a14d78b74f7e33a17dc8529d6326199 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Wed, 18 May 2022 17:45:03 -0400 Subject: [PATCH 3/6] Adding Python 3.10 to test matrix (#177) * Updating to test python 3.10 * Fixing mypy issues from upgrade * Extending path in init * Adding newline * Updating changelog * Adding response stubs --- .github/scripts/integration-test-matrix.js | 2 +- .github/workflows/main.yml | 4 +- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 7 +- dbt/__init__.py | 3 + dbt/adapters/bigquery/impl.py | 2 +- dev-requirements.txt | 3 +- mypy.ini | 3 +- setup.py | 1 + tests/integration/base.py | 8 ++ third-party-stubs/agate/__init__.pyi | 89 ++++++++++++++++++++++ third-party-stubs/agate/data_types.pyi | 71 +++++++++++++++++ tox.ini | 6 +- 13 files changed, 190 insertions(+), 11 deletions(-) create mode 100644 dbt/__init__.py create mode 100644 third-party-stubs/agate/__init__.pyi create mode 100644 third-party-stubs/agate/data_types.pyi diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 6a33653a2..1e3bb0f0d 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -1,6 +1,6 @@ module.exports = ({ context }) => { const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.7", "3.8", "3.9"]; + const supportedPythonVersions = ["3.7", "3.8", "3.9", "3.10"]; const supportedAdapters = ["bigquery"]; // if PR, generate matrix based on files changed and PR labels diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d2ed07654..2e961dfa7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,7 +69,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8] # TODO: support unit testing for python 3.9 (https://github.com/dbt-labs/dbt/issues/3689) + python-version: ['3.7', '3.8', '3.9', '3.10'] env: TOXENV: "unit" @@ -173,7 +173,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9] + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - name: Set up Python ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d247581b..cf4ddf68a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: alias: flake8-check stages: [manual] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v0.942 hooks: - id: mypy args: [--show-error-codes, --ignore-missing-imports] diff --git a/CHANGELOG.md b/CHANGELOG.md index 528cd08b1..be153ce60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ -## dbt-bigquery 1.1.0 (Release TBD) +## dbt-bigquery 1.2.0 (Release TBD) +- Adding Python 3.10 testing and enabling mypy ([#177](https://github.com/dbt-labs/dbt-bigquery/pull/177)) + +## dbt-bigquery 1.1.0 (April 28, 2022) + +## dbt-bigquery 1.1.0rc2 (April 20, 2022) ### Fixes - Restore default behavior for query timeout. Set default `job_execution_timeout` to `None` by default. Keep 300 seconds as query timeout where previously used. diff --git a/dbt/__init__.py b/dbt/__init__.py new file mode 100644 index 000000000..b36383a61 --- /dev/null +++ b/dbt/__init__.py @@ -0,0 +1,3 @@ +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 50ca3c6e1..344ffb22e 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -296,7 +296,7 @@ def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str: @classmethod def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: - decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) + decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) # type: ignore[attr-defined] return "float64" if decimals else "int64" @classmethod diff --git a/dev-requirements.txt b/dev-requirements.txt index ff7b6522b..09cf7f7c2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,7 +9,7 @@ flake8 flaky freezegun==1.1.0 ipdb -mypy==0.782 +mypy==0.942 pip-tools pre-commit pytest @@ -19,5 +19,6 @@ pytest-csv pytest-xdist pytz tox>=3.13 +types-requests twine wheel diff --git a/mypy.ini b/mypy.ini index 51fada1b1..b111482fc 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,3 +1,4 @@ [mypy] -mypy_path = ./third-party-stubs +mypy_path = third-party-stubs/ namespace_packages = True +exclude = third-party-stubs/* diff --git a/setup.py b/setup.py index 461e9746b..0adf95f3b 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ def _get_dbt_core_version(): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], python_requires=">=3.7", ) diff --git a/tests/integration/base.py b/tests/integration/base.py index 243ac92d8..ec6769e8e 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -7,6 +7,7 @@ import tempfile import traceback import unittest +import warnings from contextlib import contextmanager from datetime import datetime from functools import wraps @@ -230,6 +231,13 @@ def _generate_test_root_dir(self): return normalize(tempfile.mkdtemp(prefix='dbt-int-test-')) def setUp(self): + # Logbook warnings are ignored so we don't have to fork logbook to support python 3.10. + # This _only_ works for tests in `test/integration`. + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + module="logbook" + ) self.dbt_core_install_root = os.path.dirname(dbt.__file__) log_manager.reset_handlers() self.initial_dir = INITIAL_ROOT diff --git a/third-party-stubs/agate/__init__.pyi b/third-party-stubs/agate/__init__.pyi new file mode 100644 index 000000000..c773cc7d7 --- /dev/null +++ b/third-party-stubs/agate/__init__.pyi @@ -0,0 +1,89 @@ +from collections.abc import Sequence + +from typing import Any, Optional, Callable, Iterable, Dict, Union + +from . import data_types as data_types +from .data_types import ( + Text as Text, + Number as Number, + Boolean as Boolean, + DateTime as DateTime, + Date as Date, + TimeDelta as TimeDelta, +) + +class MappedSequence(Sequence): + def __init__(self, values: Any, keys: Optional[Any] = ...) -> None: ... + def __unicode__(self): ... + def __getitem__(self, key: Any): ... + def __setitem__(self, key: Any, value: Any) -> None: ... + def __iter__(self): ... + def __len__(self): ... + def __eq__(self, other: Any): ... + def __ne__(self, other: Any): ... + def __contains__(self, value: Any): ... + def keys(self): ... + def values(self): ... + def items(self): ... + def get(self, key: Any, default: Optional[Any] = ...): ... + def dict(self): ... + +class Row(MappedSequence): ... + +class Table: + def __init__( + self, + rows: Any, + column_names: Optional[Any] = ..., + column_types: Optional[Any] = ..., + row_names: Optional[Any] = ..., + _is_fork: bool = ..., + ) -> None: ... + def __len__(self): ... + def __iter__(self): ... + def __getitem__(self, key: Any): ... + @property + def column_types(self): ... + @property + def column_names(self): ... + @property + def row_names(self): ... + @property + def columns(self): ... + @property + def rows(self): ... + def print_csv(self, **kwargs: Any) -> None: ... + def print_json(self, **kwargs: Any) -> None: ... + def where(self, test: Callable[[Row], bool]) -> "Table": ... + def select(self, key: Union[Iterable[str], str]) -> "Table": ... + # these definitions are much narrower than what's actually accepted + @classmethod + def from_object( + cls, obj: Iterable[Dict[str, Any]], *, column_types: Optional["TypeTester"] = None + ) -> "Table": ... + @classmethod + def from_csv( + cls, path: Iterable[str], *, column_types: Optional["TypeTester"] = None + ) -> "Table": ... + @classmethod + def merge(cls, tables: Iterable["Table"]) -> "Table": ... + def rename( + self, + column_names: Optional[Iterable[str]] = None, + row_names: Optional[Any] = None, + slug_columns: bool = False, + slug_rows: bool = False, + **kwargs: Any, + ) -> "Table": ... + +class TypeTester: + def __init__( + self, force: Any = ..., limit: Optional[Any] = ..., types: Optional[Any] = ... + ) -> None: ... + def run(self, rows: Any, column_names: Any): ... + +class MaxPrecision: + def __init__(self, column_name: Any) -> None: ... + +# this is not strictly true, but it's all we care about. +def aggregate(self, aggregations: MaxPrecision) -> int: ... diff --git a/third-party-stubs/agate/data_types.pyi b/third-party-stubs/agate/data_types.pyi new file mode 100644 index 000000000..8114f7b55 --- /dev/null +++ b/third-party-stubs/agate/data_types.pyi @@ -0,0 +1,71 @@ +from typing import Any, Optional + +DEFAULT_NULL_VALUES: Any + +class DataType: + null_values: Any = ... + def __init__(self, null_values: Any = ...) -> None: ... + def test(self, d: Any): ... + def cast(self, d: Any) -> None: ... + def csvify(self, d: Any): ... + def jsonify(self, d: Any): ... + +DEFAULT_TRUE_VALUES: Any +DEFAULT_FALSE_VALUES: Any + +class Boolean(DataType): + true_values: Any = ... + false_values: Any = ... + def __init__( + self, true_values: Any = ..., false_values: Any = ..., null_values: Any = ... + ) -> None: ... + def cast(self, d: Any): ... + def jsonify(self, d: Any): ... + +ZERO_DT: Any + +class Date(DataType): + date_format: Any = ... + parser: Any = ... + def __init__(self, date_format: Optional[Any] = ..., **kwargs: Any) -> None: ... + def cast(self, d: Any): ... + def csvify(self, d: Any): ... + def jsonify(self, d: Any): ... + +class DateTime(DataType): + datetime_format: Any = ... + timezone: Any = ... + def __init__( + self, datetime_format: Optional[Any] = ..., timezone: Optional[Any] = ..., **kwargs: Any + ) -> None: ... + def cast(self, d: Any): ... + def csvify(self, d: Any): ... + def jsonify(self, d: Any): ... + +DEFAULT_CURRENCY_SYMBOLS: Any +POSITIVE: Any +NEGATIVE: Any + +class Number(DataType): + locale: Any = ... + currency_symbols: Any = ... + group_symbol: Any = ... + decimal_symbol: Any = ... + def __init__( + self, + locale: str = ..., + group_symbol: Optional[Any] = ..., + decimal_symbol: Optional[Any] = ..., + currency_symbols: Any = ..., + **kwargs: Any, + ) -> None: ... + def cast(self, d: Any): ... + def jsonify(self, d: Any): ... + +class TimeDelta(DataType): + def cast(self, d: Any): ... + +class Text(DataType): + cast_nulls: Any = ... + def __init__(self, cast_nulls: bool = ..., **kwargs: Any) -> None: ... + def cast(self, d: Any): ... diff --git a/tox.ini b/tox.ini index a5da0d0a6..348be15af 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py37,py38,py39 +envlist = py37,py38,py39,py310 -[testenv:{unit,py37,py38,py39,py}] +[testenv:{unit,py37,py38,py39,py310,py}] description = unit testing skip_install = true passenv = DBT_* PYTEST_ADDOPTS @@ -11,7 +11,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py37,py38,py39,py}-{bigquery}] +[testenv:{integration,py37,py38,py39,py310,py}-{bigquery}] description = adapter plugin integration testing skip_install = true passenv = DBT_* BIGQUERY_TEST_* PYTEST_ADDOPTS From 69d1bf211199202642e0b3965b00f70f1d77e7b2 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 23 May 2022 18:39:06 -0400 Subject: [PATCH 4/6] Implement TestDocsGenerateBigQuery test (#190) * Implement TestDocsGenerateBigQuery test * Remove legacy docs_generate tests * Add -vv to get info on failing test --- tests/functional/adapter/expected_stats.py | 62 + tests/functional/adapter/test_basic.py | 23 +- .../assets/lorem-ipsum.txt | 1 - .../bq_models/clustered.sql | 9 - .../bq_models/multi_clustered.sql | 9 - .../bq_models/nested_table.sql | 15 - .../bq_models/nested_view.sql | 7 - .../docs_generate_tests/bq_models/schema.yml | 44 - .../bq_models_noschema/disabled.sql | 2 - .../bq_models_noschema/model.sql | 2 - .../cross_db_models/model.sql | 8 - .../cross_db_models/schema.yml | 21 - .../fail_macros/failure.sql | 3 - .../docs_generate_tests/macros/dummy_test.sql | 9 - .../docs_generate_tests/macros/macro.md | 8 - .../docs_generate_tests/macros/schema.yml | 10 - .../docs_generate_tests/models/model.sql | 8 - .../docs_generate_tests/models/readme.md | 1 - .../docs_generate_tests/models/schema.yml | 82 - .../models/second_model.sql | 8 - .../docs_generate_tests/ref_models/docs.md | 31 - .../ref_models/ephemeral_copy.sql | 7 - .../ref_models/ephemeral_summary.sql | 9 - .../docs_generate_tests/ref_models/schema.yml | 48 - .../ref_models/view_summary.sql | 8 - .../docs_generate_tests/seed/schema.yml | 15 - .../docs_generate_tests/seed/seed.csv | 2 - .../snapshot/snapshot_seed.sql | 11 - .../docs_generate_tests/test_docs_generate.py | 1988 ----------------- .../trivial_models/model.sql | 1 - tox.ini | 2 +- 31 files changed, 85 insertions(+), 2369 deletions(-) create mode 100644 tests/functional/adapter/expected_stats.py delete mode 100644 tests/integration/docs_generate_tests/assets/lorem-ipsum.txt delete mode 100644 tests/integration/docs_generate_tests/bq_models/clustered.sql delete mode 100644 tests/integration/docs_generate_tests/bq_models/multi_clustered.sql delete mode 100644 tests/integration/docs_generate_tests/bq_models/nested_table.sql delete mode 100644 tests/integration/docs_generate_tests/bq_models/nested_view.sql delete mode 100644 tests/integration/docs_generate_tests/bq_models/schema.yml delete mode 100644 tests/integration/docs_generate_tests/bq_models_noschema/disabled.sql delete mode 100644 tests/integration/docs_generate_tests/bq_models_noschema/model.sql delete mode 100644 tests/integration/docs_generate_tests/cross_db_models/model.sql delete mode 100644 tests/integration/docs_generate_tests/cross_db_models/schema.yml delete mode 100644 tests/integration/docs_generate_tests/fail_macros/failure.sql delete mode 100644 tests/integration/docs_generate_tests/macros/dummy_test.sql delete mode 100644 tests/integration/docs_generate_tests/macros/macro.md delete mode 100644 tests/integration/docs_generate_tests/macros/schema.yml delete mode 100644 tests/integration/docs_generate_tests/models/model.sql delete mode 100644 tests/integration/docs_generate_tests/models/readme.md delete mode 100644 tests/integration/docs_generate_tests/models/schema.yml delete mode 100644 tests/integration/docs_generate_tests/models/second_model.sql delete mode 100644 tests/integration/docs_generate_tests/ref_models/docs.md delete mode 100644 tests/integration/docs_generate_tests/ref_models/ephemeral_copy.sql delete mode 100644 tests/integration/docs_generate_tests/ref_models/ephemeral_summary.sql delete mode 100644 tests/integration/docs_generate_tests/ref_models/schema.yml delete mode 100644 tests/integration/docs_generate_tests/ref_models/view_summary.sql delete mode 100644 tests/integration/docs_generate_tests/seed/schema.yml delete mode 100644 tests/integration/docs_generate_tests/seed/seed.csv delete mode 100644 tests/integration/docs_generate_tests/snapshot/snapshot_seed.sql delete mode 100644 tests/integration/docs_generate_tests/test_docs_generate.py delete mode 100644 tests/integration/docs_generate_tests/trivial_models/model.sql diff --git a/tests/functional/adapter/expected_stats.py b/tests/functional/adapter/expected_stats.py new file mode 100644 index 000000000..f4a1e022d --- /dev/null +++ b/tests/functional/adapter/expected_stats.py @@ -0,0 +1,62 @@ +from dbt.tests.util import AnyString, AnyFloat + + +def bigquery_stats(is_table, partition=None, cluster=None): + stats = {} + + if is_table: + stats.update( + { + "num_bytes": { + "id": "num_bytes", + "label": AnyString(), + "value": AnyFloat(), + "description": AnyString(), + "include": True, + }, + "num_rows": { + "id": "num_rows", + "label": AnyString(), + "value": AnyFloat(), + "description": AnyString(), + "include": True, + }, + } + ) + + if partition is not None: + stats.update( + { + "partitioning_type": { + "id": "partitioning_type", + "label": AnyString(), + "value": partition, + "description": AnyString(), + "include": True, + } + } + ) + + if cluster is not None: + stats.update( + { + "clustering_fields": { + "id": "clustering_fields", + "label": AnyString(), + "value": cluster, + "description": AnyString(), + "include": True, + } + } + ) + + has_stats = { + "id": "has_stats", + "label": "Has Stats?", + "value": bool(stats), + "description": "Indicates whether there are statistics for this table", + "include": False, + } + stats["has_stats"] = has_stats + + return stats diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 2c07a6e98..f95f043cf 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -13,6 +13,9 @@ from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection +from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate +from dbt.tests.adapter.basic.expected_catalog import base_expected_catalog +from tests.functional.adapter.expected_stats import bigquery_stats class TestSimpleMaterializationsBigQuery(BaseSimpleMaterializations): @@ -53,8 +56,26 @@ class TestSnapshotCheckColsBigQuery(BaseSnapshotCheckCols): class TestSnapshotTimestampBigQuery(BaseSnapshotTimestamp): pass + class TestBaseAdapterMethodBigQuery(BaseAdapterMethod): pass + class TestBigQueryValidateConnection(BaseValidateConnection): - pass \ No newline at end of file + pass + + +class TestDocsGenerateBigQuery(BaseDocsGenerate): + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return base_expected_catalog( + project, + role=None, + id_type="INT64", + text_type="STRING", + time_type="DATETIME", + view_type="view", + table_type="table", + model_stats=bigquery_stats(False), + seed_stats=bigquery_stats(True), + ) diff --git a/tests/integration/docs_generate_tests/assets/lorem-ipsum.txt b/tests/integration/docs_generate_tests/assets/lorem-ipsum.txt deleted file mode 100644 index cee7a927c..000000000 --- a/tests/integration/docs_generate_tests/assets/lorem-ipsum.txt +++ /dev/null @@ -1 +0,0 @@ -Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. diff --git a/tests/integration/docs_generate_tests/bq_models/clustered.sql b/tests/integration/docs_generate_tests/bq_models/clustered.sql deleted file mode 100644 index 744d2ecb2..000000000 --- a/tests/integration/docs_generate_tests/bq_models/clustered.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized='table', - partition_by={'field': 'updated_at', 'data_type': 'date'}, - cluster_by=['first_name'] - ) -}} - -select id,first_name,email,ip_address,DATE(updated_at) as updated_at from {{ ref('seed') }} diff --git a/tests/integration/docs_generate_tests/bq_models/multi_clustered.sql b/tests/integration/docs_generate_tests/bq_models/multi_clustered.sql deleted file mode 100644 index e47df02f9..000000000 --- a/tests/integration/docs_generate_tests/bq_models/multi_clustered.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized='table', - partition_by={'field': 'updated_at', 'data_type': 'date'}, - cluster_by=['first_name','email'] - ) -}} - -select id,first_name,email,ip_address,DATE(updated_at) as updated_at from {{ ref('seed') }} diff --git a/tests/integration/docs_generate_tests/bq_models/nested_table.sql b/tests/integration/docs_generate_tests/bq_models/nested_table.sql deleted file mode 100644 index 22f9048d6..000000000 --- a/tests/integration/docs_generate_tests/bq_models/nested_table.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -select - 1 as field_1, - 2 as field_2, - 3 as field_3, - - struct( - 5 as field_5, - 6 as field_6 - ) as nested_field diff --git a/tests/integration/docs_generate_tests/bq_models/nested_view.sql b/tests/integration/docs_generate_tests/bq_models/nested_view.sql deleted file mode 100644 index 29e7b4a2d..000000000 --- a/tests/integration/docs_generate_tests/bq_models/nested_view.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ - config( - materialized='view' - ) -}} - -select * from {{ ref('nested_table') }} diff --git a/tests/integration/docs_generate_tests/bq_models/schema.yml b/tests/integration/docs_generate_tests/bq_models/schema.yml deleted file mode 100644 index 87ebd8c71..000000000 --- a/tests/integration/docs_generate_tests/bq_models/schema.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: 2 - -models: - - name: nested_view - description: "The test model" - columns: - - name: field_1 - description: The first field - - name: field_2 - description: The second field - - name: field_3 - description: The third field - - name: nested_field.field_4 - description: The first nested field - - name: nested_field.field_5 - description: The second nested field - - name: clustered - description: "A clustered and partitioned copy of the test model" - columns: - - name: id - description: The user id - - name: first_name - description: The user's name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: When the user was updated - - name: multi_clustered - description: "A clustered and partitioned copy of the test model, clustered on multiple columns" - columns: - - name: id - description: The user id - - name: first_name - description: The user's name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: When the user was updated - - diff --git a/tests/integration/docs_generate_tests/bq_models_noschema/disabled.sql b/tests/integration/docs_generate_tests/bq_models_noschema/disabled.sql deleted file mode 100644 index e4368a859..000000000 --- a/tests/integration/docs_generate_tests/bq_models_noschema/disabled.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(disabled=true, schema='notrealnotreal') }} -select 1 as id diff --git a/tests/integration/docs_generate_tests/bq_models_noschema/model.sql b/tests/integration/docs_generate_tests/bq_models_noschema/model.sql deleted file mode 100644 index 2fb872e84..000000000 --- a/tests/integration/docs_generate_tests/bq_models_noschema/model.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(schema=var('extra_schema')) }} -select 1 as id diff --git a/tests/integration/docs_generate_tests/cross_db_models/model.sql b/tests/integration/docs_generate_tests/cross_db_models/model.sql deleted file mode 100644 index 0759a4572..000000000 --- a/tests/integration/docs_generate_tests/cross_db_models/model.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized='view', - database=var('alternate_db') - ) -}} - -select * from {{ ref('seed') }} diff --git a/tests/integration/docs_generate_tests/cross_db_models/schema.yml b/tests/integration/docs_generate_tests/cross_db_models/schema.yml deleted file mode 100644 index 6f3fc6619..000000000 --- a/tests/integration/docs_generate_tests/cross_db_models/schema.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 2 - -models: - - name: model - description: "The test model" - columns: - - name: id - description: The user ID number - tests: - - unique - - not_null - - name: first_name - description: The user's first name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: The last time this user's email was updated - tests: - - test.nothing diff --git a/tests/integration/docs_generate_tests/fail_macros/failure.sql b/tests/integration/docs_generate_tests/fail_macros/failure.sql deleted file mode 100644 index f0519ed24..000000000 --- a/tests/integration/docs_generate_tests/fail_macros/failure.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro get_catalog(information_schema, schemas) %} - {% do exceptions.raise_compiler_error('rejected: no catalogs for you') %} -{% endmacro %} diff --git a/tests/integration/docs_generate_tests/macros/dummy_test.sql b/tests/integration/docs_generate_tests/macros/dummy_test.sql deleted file mode 100644 index adf4d7d93..000000000 --- a/tests/integration/docs_generate_tests/macros/dummy_test.sql +++ /dev/null @@ -1,9 +0,0 @@ - -{% test nothing(model) %} - --- a silly test to make sure that table-level tests show up in the manifest --- without a column_name field -select 0 - -{% endtest %} - diff --git a/tests/integration/docs_generate_tests/macros/macro.md b/tests/integration/docs_generate_tests/macros/macro.md deleted file mode 100644 index 6a48ffc1b..000000000 --- a/tests/integration/docs_generate_tests/macros/macro.md +++ /dev/null @@ -1,8 +0,0 @@ - -{% docs macro_info %} -My custom test that I wrote that does nothing -{% enddocs %} - -{% docs macro_arg_info %} -The model for my custom test -{% enddocs %} diff --git a/tests/integration/docs_generate_tests/macros/schema.yml b/tests/integration/docs_generate_tests/macros/schema.yml deleted file mode 100644 index 6b33f4730..000000000 --- a/tests/integration/docs_generate_tests/macros/schema.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 -macros: - - name: test_nothing - description: "{{ doc('macro_info') }}" - meta: - some_key: 100 - arguments: - - name: model - type: Relation - description: "{{ doc('macro_arg_info') }}" diff --git a/tests/integration/docs_generate_tests/models/model.sql b/tests/integration/docs_generate_tests/models/model.sql deleted file mode 100644 index 0759a4572..000000000 --- a/tests/integration/docs_generate_tests/models/model.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized='view', - database=var('alternate_db') - ) -}} - -select * from {{ ref('seed') }} diff --git a/tests/integration/docs_generate_tests/models/readme.md b/tests/integration/docs_generate_tests/models/readme.md deleted file mode 100644 index d59a7f447..000000000 --- a/tests/integration/docs_generate_tests/models/readme.md +++ /dev/null @@ -1 +0,0 @@ -This is a readme.md file with {{ invalid-ish jinja }} in it diff --git a/tests/integration/docs_generate_tests/models/schema.yml b/tests/integration/docs_generate_tests/models/schema.yml deleted file mode 100644 index 600be50ea..000000000 --- a/tests/integration/docs_generate_tests/models/schema.yml +++ /dev/null @@ -1,82 +0,0 @@ -version: 2 - -models: - - name: model - description: "The test model" - docs: - show: false - columns: - - name: id - description: The user ID number - tests: - - unique - - not_null - - name: first_name - description: The user's first name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: The last time this user's email was updated - tests: - - test.nothing - - - name: second_model - description: "The second test model" - docs: - show: false - columns: - - name: id - description: The user ID number - - name: first_name - description: The user's first name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: The last time this user's email was updated - - -sources: - - name: my_source - description: "My source" - loader: a_loader - schema: "{{ var('test_schema') }}" - tables: - - name: my_table - description: "My table" - identifier: seed - quoting: - identifier: True - columns: - - name: id - description: "An ID field" - - -exposures: - - name: simple_exposure - type: dashboard - depends_on: - - ref('model') - - source('my_source', 'my_table') - owner: - email: something@example.com - - name: notebook_exposure - type: notebook - depends_on: - - ref('model') - - ref('second_model') - owner: - email: something@example.com - name: Some name - description: > - A description of the complex exposure - maturity: medium - meta: - tool: 'my_tool' - languages: - - python - tags: ['my_department'] - url: http://example.com/notebook/1 diff --git a/tests/integration/docs_generate_tests/models/second_model.sql b/tests/integration/docs_generate_tests/models/second_model.sql deleted file mode 100644 index 37f83155c..000000000 --- a/tests/integration/docs_generate_tests/models/second_model.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized='view', - schema='test', - ) -}} - -select * from {{ ref('seed') }} diff --git a/tests/integration/docs_generate_tests/ref_models/docs.md b/tests/integration/docs_generate_tests/ref_models/docs.md deleted file mode 100644 index c5ad96862..000000000 --- a/tests/integration/docs_generate_tests/ref_models/docs.md +++ /dev/null @@ -1,31 +0,0 @@ -{% docs ephemeral_summary %} -A summmary table of the ephemeral copy of the seed data -{% enddocs %} - -{% docs summary_first_name %} -The first name being summarized -{% enddocs %} - -{% docs summary_count %} -The number of instances of the first name -{% enddocs %} - -{% docs view_summary %} -A view of the summary of the ephemeral copy of the seed data -{% enddocs %} - -{% docs source_info %} -My source -{% enddocs %} - -{% docs table_info %} -My table -{% enddocs %} - -{% docs column_info %} -An ID field -{% enddocs %} - -{% docs notebook_info %} -A description of the complex exposure -{% enddocs %} diff --git a/tests/integration/docs_generate_tests/ref_models/ephemeral_copy.sql b/tests/integration/docs_generate_tests/ref_models/ephemeral_copy.sql deleted file mode 100644 index 3f7e698ce..000000000 --- a/tests/integration/docs_generate_tests/ref_models/ephemeral_copy.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ - config( - materialized = "ephemeral" - ) -}} - -select * from {{ source("my_source", "my_table") }} diff --git a/tests/integration/docs_generate_tests/ref_models/ephemeral_summary.sql b/tests/integration/docs_generate_tests/ref_models/ephemeral_summary.sql deleted file mode 100644 index 4dba8a564..000000000 --- a/tests/integration/docs_generate_tests/ref_models/ephemeral_summary.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized = "table" - ) -}} - -select first_name, count(*) as ct from {{ref('ephemeral_copy')}} -group by first_name -order by first_name asc diff --git a/tests/integration/docs_generate_tests/ref_models/schema.yml b/tests/integration/docs_generate_tests/ref_models/schema.yml deleted file mode 100644 index 6d7dcea05..000000000 --- a/tests/integration/docs_generate_tests/ref_models/schema.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: 2 - -models: - - name: ephemeral_summary - description: "{{ doc('ephemeral_summary') }}" - columns: &summary_columns - - name: first_name - description: "{{ doc('summary_first_name') }}" - - name: ct - description: "{{ doc('summary_count') }}" - - name: view_summary - description: "{{ doc('view_summary') }}" - columns: *summary_columns - -sources: - - name: my_source - description: "{{ doc('source_info') }}" - loader: a_loader - schema: "{{ var('test_schema') }}" - quoting: - database: False - identifier: False - tables: - - name: my_table - description: "{{ doc('table_info') }}" - identifier: seed - quoting: - identifier: True - columns: - - name: id - description: "{{ doc('column_info') }}" - -exposures: - - name: notebook_exposure - type: notebook - depends_on: - - ref('view_summary') - owner: - email: something@example.com - name: Some name - description: "{{ doc('notebook_info') }}" - maturity: medium - url: http://example.com/notebook/1 - meta: - tool: 'my_tool' - languages: - - python - tags: ['my_department'] diff --git a/tests/integration/docs_generate_tests/ref_models/view_summary.sql b/tests/integration/docs_generate_tests/ref_models/view_summary.sql deleted file mode 100644 index 6ad6c3dd0..000000000 --- a/tests/integration/docs_generate_tests/ref_models/view_summary.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized = "view" - ) -}} - -select first_name, ct from {{ref('ephemeral_summary')}} -order by ct asc diff --git a/tests/integration/docs_generate_tests/seed/schema.yml b/tests/integration/docs_generate_tests/seed/schema.yml deleted file mode 100644 index ef5e7dc9e..000000000 --- a/tests/integration/docs_generate_tests/seed/schema.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: 2 -seeds: - - name: seed - description: "The test seed" - columns: - - name: id - description: The user ID number - - name: first_name - description: The user's first name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: The last time this user's email was updated diff --git a/tests/integration/docs_generate_tests/seed/seed.csv b/tests/integration/docs_generate_tests/seed/seed.csv deleted file mode 100644 index ef154f552..000000000 --- a/tests/integration/docs_generate_tests/seed/seed.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,first_name,email,ip_address,updated_at -1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 diff --git a/tests/integration/docs_generate_tests/snapshot/snapshot_seed.sql b/tests/integration/docs_generate_tests/snapshot/snapshot_seed.sql deleted file mode 100644 index 83bc54fe7..000000000 --- a/tests/integration/docs_generate_tests/snapshot/snapshot_seed.sql +++ /dev/null @@ -1,11 +0,0 @@ -{% snapshot snapshot_seed %} -{{ - config( - unique_key='id', - strategy='check', - check_cols='all', - target_schema=var('alternate_schema') - ) -}} -select * from {{ ref('seed') }} -{% endsnapshot %} \ No newline at end of file diff --git a/tests/integration/docs_generate_tests/test_docs_generate.py b/tests/integration/docs_generate_tests/test_docs_generate.py deleted file mode 100644 index 0f62c025f..000000000 --- a/tests/integration/docs_generate_tests/test_docs_generate.py +++ /dev/null @@ -1,1988 +0,0 @@ -import hashlib -import json -import os -import random -import shutil -import tempfile -import time -from datetime import datetime -from unittest.mock import ANY, patch - -from pytest import mark -from tests.integration.base import DBTIntegrationTest, use_profile, AnyFloat, \ - AnyString, AnyStringWith, normalize, Normalized - -import dbt.tracking -import dbt.version -from dbt.exceptions import CompilationException - - -def _read_file(path): - with open(path, 'r') as fp: - return fp.read().replace('\r', '').replace('\\r', '') - - -class LineIndifferent: - def __init__(self, expected): - self.expected = expected.replace('\r', '') - - def __eq__(self, other): - return self.expected == other.replace('\r', '') - - def __repr__(self): - return 'LineIndifferent({!r})'.format(self.expected) - - def __str__(self): - return self.__repr__() - - -class OneOf: - def __init__(self, *options): - self.options = options - - def __eq__(self, other): - return any(o == other for o in self.options) - - def __repr__(self): - return 'OneOf(*{!r})'.format(self.options) - - def __str__(self): - return self.__repr__() - - -def _read_json(path): - # read json generated by dbt. - with open(path) as fp: - return json.load(fp) - - -def walk_files(path): - for root, dirs, files in os.walk(path): - for basename in files: - yield os.path.join(root, basename) - - -class TestDocsGenerateEscapes(DBTIntegrationTest): - prefix = "pgtest{}{:04}".format(int(time.time()), random.randint(0, 9999)) - - @property - def schema(self): - return 'docs_generate' - - @staticmethod - def dir(path): - return normalize(path) - - @property - def models(self): - return self.dir("trivial_models") - - def run_and_generate(self): - self.assertEqual(len(self.run_dbt(['run'])), 1) - os.remove(normalize('target/manifest.json')) - os.remove(normalize('target/run_results.json')) - self.run_dbt(['docs', 'generate']) - - -class TestDocsGenerate(DBTIntegrationTest): - setup_alternate_db = True - - def adapter_case(self, value): - return value.lower() - - def setUp(self): - super().setUp() - self.maxDiff = None - self.alternate_schema = self.unique_schema() + '_test' - - self._created_schemas.add(self.alternate_schema) - os.environ['DBT_ENV_CUSTOM_ENV_env_key'] = 'env_value' - - def tearDown(self): - super().tearDown() - del os.environ['DBT_ENV_CUSTOM_ENV_env_key'] - - @property - def schema(self): - return 'docs_generate' - - @staticmethod - def dir(path): - return normalize(path) - - @property - def models(self): - return self.dir("models") - - @property - def project_config(self): - return { - 'config-version': 2, - 'quoting': { - 'identifier': False - } - } - - def run_and_generate(self, extra=None, seed_count=1, model_count=2, alternate_db=None, args=None): - if alternate_db is None: - alternate_db = self.alternative_database - project = { - "seed-paths": [self.dir("seed")], - 'macro-paths': [self.dir('macros')], - 'snapshot-paths': [self.dir('snapshot')], - 'vars': { - 'alternate_db': alternate_db, - 'alternate_schema': self.alternate_schema, - }, - 'seeds': { - 'quote_columns': True, - }, - } - if extra: - project.update(extra) - self.use_default_project(project) - - vars_arg = '--vars={{test_schema: {}}}'.format(self.unique_schema()) - - self.assertEqual(len(self.run_dbt(["seed", vars_arg])), seed_count) - self.assertEqual(len(self.run_dbt(['run', vars_arg])), model_count) - os.remove(normalize('target/manifest.json')) - os.remove(normalize('target/run_results.json')) - self.generate_start_time = datetime.utcnow() - base_args = ['docs', 'generate', vars_arg] - if args: - base_args.extend(args) - self.run_dbt(base_args) - - def _no_stats(self): - return { - 'has_stats': { - 'id': 'has_stats', - 'label': 'Has Stats?', - 'value': False, - 'description': 'Indicates whether there are statistics for this table', - 'include': False, - }, - } - - def _bigquery_stats(self, is_table, partition=None, cluster=None): - stats = {} - - if is_table: - stats.update({ - 'num_bytes': { - 'id': 'num_bytes', - 'label': AnyString(), - 'value': AnyFloat(), - 'description': AnyString(), - 'include': True, - }, - 'num_rows': { - 'id': 'num_rows', - 'label': AnyString(), - 'value': AnyFloat(), - 'description': AnyString(), - 'include': True, - } - }) - - if partition is not None: - stats.update({ - 'partitioning_type': { - 'id': 'partitioning_type', - 'label': AnyString(), - 'value': partition, - 'description': AnyString(), - 'include': True - } - }) - - if cluster is not None: - stats.update({ - 'clustering_fields': { - 'id': 'clustering_fields', - 'label': AnyString(), - 'value': cluster, - 'description': AnyString(), - 'include': True - } - }) - - has_stats = { - 'id': 'has_stats', - 'label': 'Has Stats?', - 'value': bool(stats), - 'description': 'Indicates whether there are statistics for this table', - 'include': False, - } - stats['has_stats'] = has_stats - - return stats - - def _expected_catalog(self, id_type, text_type, time_type, view_type, - table_type, model_stats, seed_stats=None, case=None, - case_columns=False, model_database=None): - if case is None: - def case(x): return x - col_case = case if case_columns else lambda x: x - - if seed_stats is None: - seed_stats = model_stats - - if model_database is None: - model_database = self.default_database - my_schema_name = self.unique_schema() - role = self.get_role() - expected_cols = { - col_case('id'): { - 'name': col_case('id'), - 'index': 1, - 'type': id_type, - 'comment': None, - }, - col_case('first_name'): { - 'name': col_case('first_name'), - 'index': 2, - 'type': text_type, - 'comment': None, - }, - col_case('email'): { - 'name': col_case('email'), - 'index': 3, - 'type': text_type, - 'comment': None, - }, - col_case('ip_address'): { - 'name': col_case('ip_address'), - 'index': 4, - 'type': text_type, - 'comment': None, - }, - col_case('updated_at'): { - 'name': col_case('updated_at'), - 'index': 5, - 'type': time_type, - 'comment': None, - }, - } - return { - 'nodes': { - 'model.test.model': { - 'unique_id': 'model.test.model', - 'metadata': { - 'schema': my_schema_name, - 'database': model_database, - 'name': case('model'), - 'type': view_type, - 'comment': None, - 'owner': role, - }, - 'stats': model_stats, - 'columns': expected_cols, - }, - 'model.test.second_model': { - 'unique_id': 'model.test.second_model', - 'metadata': { - 'schema': self.alternate_schema, - 'database': self.default_database, - 'name': case('second_model'), - 'type': view_type, - 'comment': None, - 'owner': role, - }, - 'stats': model_stats, - 'columns': expected_cols, - }, - 'seed.test.seed': { - 'unique_id': 'seed.test.seed', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': case('seed'), - 'type': table_type, - 'comment': None, - 'owner': role, - }, - 'stats': seed_stats, - 'columns': expected_cols, - }, - }, - 'sources': { - 'source.test.my_source.my_table': { - 'unique_id': 'source.test.my_source.my_table', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': case('seed'), - 'type': table_type, - 'comment': None, - 'owner': role, - }, - 'stats': seed_stats, - 'columns': expected_cols, - }, - }, - } - - def get_role(self): - return None - - def expected_bigquery_catalog(self): - return self._expected_catalog( - id_type='INT64', - text_type='STRING', - time_type='DATETIME', - view_type='view', - table_type='table', - model_stats=self._bigquery_stats(False), - seed_stats=self._bigquery_stats(True), - model_database=self.alternative_database, - ) - - @staticmethod - def _clustered_bigquery_columns(update_type): - return { - 'id': { - 'comment': None, - 'index': 1, - 'name': 'id', - 'type': 'INT64', - }, - 'first_name': { - 'comment': None, - 'index': 2, - 'name': 'first_name', - 'type': 'STRING', - }, - 'email': { - 'comment': None, - 'index': 3, - 'name': 'email', - 'type': 'STRING', - }, - 'ip_address': { - 'comment': None, - 'index': 4, - 'name': 'ip_address', - 'type': 'STRING', - }, - 'updated_at': { - 'comment': None, - 'index': 5, - 'name': 'updated_at', - 'type': update_type, - }, - } - - def expected_bigquery_complex_catalog(self): - my_schema_name = self.unique_schema() - role = self.get_role() - table_stats = self._bigquery_stats(True) - clustering_stats = self._bigquery_stats(True, partition='updated_at', - cluster='first_name') - multi_clustering_stats = self._bigquery_stats(True, partition='updated_at', - cluster='first_name, email') - nesting_columns = { - 'field_1': { - 'name': 'field_1', - 'index': 1, - 'type': 'INT64', - 'comment': None - }, - 'field_2': { - 'name': 'field_2', - 'index': 2, - 'type': 'INT64', - 'comment': None - }, - 'field_3': { - 'name': 'field_3', - 'index': 3, - 'type': 'INT64', - 'comment': None - }, - 'nested_field': { - 'name': 'nested_field', - 'index': 4, - 'type': 'STRUCT', - 'comment': None - }, - 'nested_field.field_5': { - 'name': 'nested_field.field_5', - 'index': 5, - 'type': 'INT64', - 'comment': None - }, - 'nested_field.field_6': { - 'name': 'nested_field.field_6', - 'index': 6, - 'type': 'INT64', - 'comment': None - } - } - - return { - 'nodes': { - 'model.test.clustered': { - 'unique_id': 'model.test.clustered', - 'metadata': { - 'comment': None, - 'name': 'clustered', - 'owner': None, - 'schema': my_schema_name, - 'database': self.default_database, - 'type': 'table' - }, - 'stats': clustering_stats, - 'columns': self._clustered_bigquery_columns('DATE'), - }, - 'model.test.multi_clustered': { - 'unique_id': 'model.test.multi_clustered', - 'metadata': { - 'comment': None, - 'name': 'multi_clustered', - 'owner': None, - 'schema': my_schema_name, - 'database': self.default_database, - 'type': 'table' - }, - 'stats': multi_clustering_stats, - 'columns': self._clustered_bigquery_columns('DATE'), - }, - 'seed.test.seed': { - 'unique_id': 'seed.test.seed', - 'metadata': { - 'comment': None, - 'name': 'seed', - 'owner': None, - 'schema': my_schema_name, - 'database': self.default_database, - 'type': 'table', - }, - 'stats': table_stats, - 'columns': self._clustered_bigquery_columns('DATETIME'), - }, - 'model.test.nested_view': { - 'unique_id': 'model.test.nested_view', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': 'nested_view', - 'type': 'view', - 'owner': role, - 'comment': None - }, - 'stats': self._bigquery_stats(False), - 'columns': nesting_columns, - }, - 'model.test.nested_table': { - 'unique_id': 'model.test.nested_table', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': 'nested_table', - 'type': 'table', - 'owner': role, - 'comment': None - }, - 'stats': table_stats, - 'columns': nesting_columns, - }, - }, - 'sources': {}, - } - - def verify_catalog(self, expected): - self.assertTrue(os.path.exists('./target/catalog.json')) - - catalog = _read_json('./target/catalog.json') - - assert set(catalog) == {'errors', 'metadata', 'nodes', 'sources'} - - self.verify_metadata( - catalog['metadata'], 'https://schemas.getdbt.com/dbt/catalog/v1.json') - assert not catalog['errors'] - - for key in 'nodes', 'sources': - assert catalog[key] == expected[key] - - def verify_manifest_macros(self, manifest, expected=None): - self.assertIn('macros', manifest) - if expected is None: - self._verify_generic_macro_structure(manifest) - return - for unique_id, expected_macro in expected.items(): - self.assertIn(unique_id, manifest['macros']) - actual_macro = manifest['macros'][unique_id] - self.assertEqual(expected_macro, actual_macro) - - def rendered_model_config(self, **updates): - result = { - 'database': None, - 'schema': None, - 'alias': None, - 'enabled': True, - 'materialized': 'view', - 'pre-hook': [], - 'post-hook': [], - 'column_types': {}, - 'quoting': {}, - 'tags': [], - 'persist_docs': {}, - 'full_refresh': None, - 'on_schema_change': 'ignore', - 'meta': {}, - 'unique_key': None, - } - result.update(updates) - return result - - def unrendered_model_config(self, **updates): - return updates - - def rendered_seed_config(self, **updates): - result = { - 'enabled': True, - 'materialized': 'seed', - 'persist_docs': {}, - 'pre-hook': [], - 'post-hook': [], - 'column_types': {}, - 'quoting': {}, - 'tags': [], - 'quote_columns': True, - 'full_refresh': None, - 'on_schema_change': 'ignore', - 'database': None, - 'schema': None, - 'alias': None, - 'meta': {}, - 'unique_key': None, - } - result.update(updates) - return result - - def unrendered_seed_config(self, **updates): - result = {'quote_columns': True} - result.update(updates) - return result - - def rendered_snapshot_config(self, **updates): - result = { - 'database': None, - 'schema': None, - 'alias': None, - 'enabled': True, - 'materialized': 'snapshot', - 'pre-hook': [], - 'post-hook': [], - 'column_types': {}, - 'quoting': {}, - 'tags': [], - 'persist_docs': {}, - 'full_refresh': None, - 'on_schema_change': 'ignore', - 'strategy': 'check', - 'check_cols': 'all', - 'unique_key': 'id', - 'target_schema': None, - 'meta': {}, - } - result.update(updates) - return result - - def unrendered_snapshot_config(self, **updates): - result = { - 'check_cols': 'all', - 'strategy': 'check', - 'target_schema': None, - 'unique_key': 'id' - } - result.update(updates) - return result - - def rendered_tst_config(self, **updates): - result = { - 'enabled': True, - 'materialized': 'test', - 'tags': [], - 'severity': 'ERROR', - 'store_failures': None, - 'warn_if': '!= 0', - 'error_if': '!= 0', - 'fail_calc': 'count(*)', - 'where': None, - 'limit': None, - 'database': None, - 'schema': 'dbt_test__audit', - 'alias': None, - 'meta': {}, - } - result.update(updates) - return result - - def unrendered_tst_config(self, **updates): - result = {} - result.update(updates) - return result - - def _verify_generic_macro_structure(self, manifest): - # just test a known global macro to avoid having to update this every - # time they change. - self.assertIn('macro.dbt.get_quoted_csv', manifest['macros']) - macro = manifest['macros']['macro.dbt.get_quoted_csv'] - self.assertEqual( - set(macro), - { - 'path', 'original_file_path', 'package_name', - 'root_path', 'name', 'unique_id', 'tags', 'resource_type', - 'depends_on', 'meta', 'description', 'patch_path', 'arguments', - 'macro_sql', 'docs', 'created_at', - } - ) - # Don't compare the sql, just make sure it exists - self.assertTrue(len(macro['macro_sql']) > 10) - without_sql = { - k: v for k, v in macro.items() - if k not in {'macro_sql'} - } - # Windows means we can't hard-code these. - helpers_path = Normalized('macros/materializations/models/incremental/column_helpers.sql') - root_path = Normalized(os.path.join( - self.dbt_core_install_root, 'include', 'global_project' - )) - self.assertEqual( - { - 'path': helpers_path, - 'original_file_path': helpers_path, - 'package_name': 'dbt', - 'root_path': root_path, - 'name': 'get_quoted_csv', - 'unique_id': 'macro.dbt.get_quoted_csv', - 'created_at': ANY, - 'tags': [], - 'resource_type': 'macro', - 'depends_on': {'macros': []}, - 'description': '', - 'docs': {'show': True}, - 'patch_path': None, - 'meta': {}, - 'arguments': [], - }, - without_sql, - ) - - def expected_seeded_manifest(self, model_database=None, quote_model=False): - models_path = self.dir('models') - model_sql_path = os.path.join(models_path, 'model.sql') - second_model_sql_path = os.path.join(models_path, 'second_model.sql') - model_schema_yml_path = os.path.join(models_path, 'schema.yml') - seed_schema_yml_path = os.path.join(self.dir('seed'), 'schema.yml') - seed_path = self.dir(os.path.join('seed', 'seed.csv')) - snapshot_path = self.dir(os.path.join('snapshot', 'snapshot_seed.sql')) - - my_schema_name = self.unique_schema() - test_audit_schema = my_schema_name + '_dbt_test__audit' - - if model_database is None: - model_database = self.alternative_database - - model_config = self.rendered_model_config(database=model_database) - second_config = self.rendered_model_config( - schema=self.alternate_schema[-4:]) - - unrendered_model_config = self.unrendered_model_config( - database=model_database, materialized='view') - unrendered_second_config = self.unrendered_model_config( - schema=self.alternate_schema[-4:], materialized='view') - - seed_config = self.rendered_seed_config() - unrendered_seed_config = self.unrendered_seed_config() - - test_config = self.rendered_tst_config() - unrendered_test_config = self.unrendered_tst_config() - - snapshot_config = self.rendered_snapshot_config( - target_schema=self.alternate_schema) - unrendered_snapshot_config = self.unrendered_snapshot_config( - target_schema=self.alternate_schema - ) - - quote_database = quote_schema = True - relation_name_node_format = self._relation_name_format( - quote_database, quote_schema, quote_model - ) - relation_name_source_format = self._relation_name_format( - quote_database, quote_schema, quote_identifier=True - ) - - return { - 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v5.json', - 'dbt_version': dbt.version.__version__, - 'nodes': { - 'model.test.model': { - 'compiled_path': Normalized('target/compiled/test/models/model.sql'), - 'build_path': None, - 'created_at': ANY, - 'name': 'model', - 'root_path': self.test_root_realpath, - 'relation_name': relation_name_node_format.format( - model_database, my_schema_name, 'model' - ), - 'resource_type': 'model', - 'path': 'model.sql', - 'original_file_path': model_sql_path, - 'package_name': 'test', - 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), - 'refs': [['seed']], - 'sources': [], - 'depends_on': {'nodes': ['seed.test.seed'], 'macros': []}, - 'unique_id': 'model.test.model', - 'fqn': ['test', 'model'], - 'tags': [], - 'meta': {}, - 'config': model_config, - 'schema': my_schema_name, - 'database': model_database, - 'deferred': False, - 'alias': 'model', - 'description': 'The test model', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'patch_path': 'test://' + model_schema_yml_path, - 'docs': {'show': False}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(model_sql_path), - 'unrendered_config': unrendered_model_config, - }, - 'model.test.second_model': { - 'compiled_path': Normalized('target/compiled/test/models/second_model.sql'), - 'build_path': None, - 'created_at': ANY, - 'name': 'second_model', - 'root_path': self.test_root_realpath, - 'relation_name': relation_name_node_format.format( - self.default_database, self.alternate_schema, - 'second_model' - ), - 'resource_type': 'model', - 'path': 'second_model.sql', - 'original_file_path': second_model_sql_path, - 'package_name': 'test', - 'raw_sql': LineIndifferent(_read_file(second_model_sql_path).rstrip('\r\n')), - 'refs': [['seed']], - 'sources': [], - 'depends_on': {'nodes': ['seed.test.seed'], 'macros': []}, - 'unique_id': 'model.test.second_model', - 'fqn': ['test', 'second_model'], - 'tags': [], - 'meta': {}, - 'config': second_config, - 'schema': self.alternate_schema, - 'database': self.default_database, - 'deferred': False, - 'alias': 'second_model', - 'description': 'The second test model', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'patch_path': 'test://' + model_schema_yml_path, - 'docs': {'show': False}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(second_model_sql_path), - 'unrendered_config': unrendered_second_config - }, - 'seed.test.seed': { - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'compiled': True, - 'compiled_sql': '', - 'config': seed_config, - 'patch_path': 'test://' + seed_schema_yml_path, - 'path': 'seed.csv', - 'name': 'seed', - 'root_path': self.test_root_realpath, - 'relation_name': relation_name_node_format.format( - self.default_database, my_schema_name, 'seed' - ), - 'resource_type': 'seed', - 'raw_sql': '', - 'package_name': 'test', - 'original_file_path': seed_path, - 'refs': [], - 'sources': [], - 'depends_on': {'nodes': [], 'macros': []}, - 'unique_id': 'seed.test.seed', - 'fqn': ['test', 'seed'], - 'tags': [], - 'meta': {}, - 'schema': my_schema_name, - 'database': self.default_database, - 'alias': 'seed', - 'deferred': False, - 'description': 'The test seed', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': '', - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(seed_path), - 'unrendered_config': unrendered_seed_config, - }, - 'test.test.not_null_model_id.d01cc630e6': { - 'alias': 'not_null_model_id', - 'compiled_path': Normalized('target/compiled/test/models/schema.yml/not_null_model_id.sql'), - 'build_path': None, - 'created_at': ANY, - 'column_name': 'id', - 'columns': {}, - 'config': test_config, - 'sources': [], - 'depends_on': { - 'macros': ['macro.dbt.test_not_null', 'macro.dbt.get_where_subquery'], - 'nodes': ['model.test.model'], - }, - 'deferred': False, - 'description': '', - 'file_key_name': 'models.model', - 'fqn': ['test', 'not_null_model_id'], - 'name': 'not_null_model_id', - 'original_file_path': model_schema_yml_path, - 'package_name': 'test', - 'patch_path': None, - 'path': Normalized('not_null_model_id.sql'), - 'raw_sql': "{{ test_not_null(**_dbt_generic_test_kwargs) }}", - 'refs': [['model']], - 'relation_name': None, - 'resource_type': 'test', - 'root_path': self.test_root_realpath, - 'schema': test_audit_schema, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'test.test.not_null_model_id.d01cc630e6', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': AnyStringWith('where id is null'), - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'test_metadata': { - 'namespace': None, - 'name': 'not_null', - 'kwargs': { - 'column_name': 'id', - 'model': "{{ get_where_subquery(ref('model')) }}", - }, - }, - 'checksum': {'name': 'none', 'checksum': ''}, - 'unrendered_config': unrendered_test_config, - }, - 'snapshot.test.snapshot_seed': { - 'alias': 'snapshot_seed', - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'checksum': self._checksum_file(snapshot_path), - 'columns': {}, - 'compiled': True, - 'compiled_sql': ANY, - 'config': snapshot_config, - 'database': self.default_database, - 'deferred': False, - 'depends_on': { - 'macros': [], - 'nodes': ['seed.test.seed'], - }, - 'description': '', - 'docs': {'show': True}, - 'extra_ctes': [], - 'extra_ctes_injected': True, - 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], - 'meta': {}, - 'name': 'snapshot_seed', - 'original_file_path': snapshot_path, - 'package_name': 'test', - 'patch_path': None, - 'path': normalize('snapshot_seed.sql'), - 'raw_sql': LineIndifferent( - _read_file(snapshot_path) - .replace('{% snapshot snapshot_seed %}', '') - .replace('{% endsnapshot %}', '')), - 'refs': [['seed']], - 'relation_name': relation_name_node_format.format( - self.default_database, self.alternate_schema, - 'snapshot_seed' - ), - 'resource_type': 'snapshot', - 'root_path': self.test_root_realpath, - 'schema': self.alternate_schema, - 'sources': [], - 'tags': [], - 'unique_id': 'snapshot.test.snapshot_seed', - 'unrendered_config': unrendered_snapshot_config, - }, - 'test.test.test_nothing_model_.5d38568946': { - 'alias': 'test_nothing_model_', - 'compiled_path': Normalized('target/compiled/test/models/schema.yml/test_nothing_model_.sql'), - 'build_path': None, - 'created_at': ANY, - 'column_name': None, - 'columns': {}, - 'config': test_config, - 'sources': [], - 'depends_on': { - 'macros': ['macro.test.test_nothing', 'macro.dbt.get_where_subquery'], - 'nodes': ['model.test.model'], - }, - 'deferred': False, - 'description': '', - 'file_key_name': 'models.model', - 'fqn': ['test', 'test_nothing_model_'], - 'name': 'test_nothing_model_', - 'original_file_path': model_schema_yml_path, - 'package_name': 'test', - 'patch_path': None, - 'path': normalize('test_nothing_model_.sql'), - 'raw_sql': "{{ test.test_nothing(**_dbt_generic_test_kwargs) }}", - 'refs': [['model']], - 'relation_name': None, - 'resource_type': 'test', - 'root_path': self.test_root_realpath, - 'schema': test_audit_schema, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'test.test.test_nothing_model_.5d38568946', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': AnyStringWith('select 0'), - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'test_metadata': { - 'namespace': 'test', - 'name': 'nothing', - 'kwargs': { - 'model': "{{ get_where_subquery(ref('model')) }}", - }, - }, - 'checksum': {'name': 'none', 'checksum': ''}, - 'unrendered_config': unrendered_test_config, - }, - 'test.test.unique_model_id.67b76558ff': { - 'alias': 'unique_model_id', - 'compiled_path': Normalized('target/compiled/test/models/schema.yml/unique_model_id.sql'), - 'build_path': None, - 'created_at': ANY, - 'column_name': 'id', - 'columns': {}, - 'config': test_config, - 'sources': [], - 'depends_on': { - 'macros': ['macro.dbt.test_unique', 'macro.dbt.get_where_subquery'], - 'nodes': ['model.test.model'], - }, - 'deferred': False, - 'description': '', - 'file_key_name': 'models.model', - 'fqn': ['test', 'unique_model_id'], - 'name': 'unique_model_id', - 'original_file_path': model_schema_yml_path, - 'package_name': 'test', - 'patch_path': None, - 'path': normalize('unique_model_id.sql'), - 'raw_sql': "{{ test_unique(**_dbt_generic_test_kwargs) }}", - 'refs': [['model']], - 'relation_name': None, - 'resource_type': 'test', - 'root_path': self.test_root_realpath, - 'schema': test_audit_schema, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'test.test.unique_model_id.67b76558ff', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': AnyStringWith('count(*)'), - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'test_metadata': { - 'namespace': None, - 'name': 'unique', - 'kwargs': { - 'column_name': 'id', - 'model': "{{ get_where_subquery(ref('model')) }}", - }, - }, - 'checksum': {'name': 'none', 'checksum': ''}, - 'unrendered_config': unrendered_test_config, - }, - }, - 'sources': { - 'source.test.my_source.my_table': { - 'created_at': ANY, - 'columns': { - 'id': { - 'description': 'An ID field', - 'name': 'id', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - } - }, - 'config': { - 'enabled': True, - }, - 'quoting': { - 'database': None, - 'schema': None, - 'identifier': True, - 'column': None, - }, - 'database': self.default_database, - 'description': 'My table', - 'external': None, - 'freshness': {'error_after': {'count': None, 'period': None}, 'warn_after': {'count': None, 'period': None}, 'filter': None}, - 'identifier': 'seed', - 'loaded_at_field': None, - 'loader': 'a_loader', - 'meta': {}, - 'name': 'my_table', - 'original_file_path': self.dir('models/schema.yml'), - 'package_name': 'test', - 'path': self.dir('models/schema.yml'), - 'patch_path': None, - 'relation_name': relation_name_source_format.format( - self.default_database, my_schema_name, 'seed' - ), - 'resource_type': 'source', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'source_description': 'My source', - 'source_name': 'my_source', - 'source_meta': {}, - 'tags': [], - 'unique_id': 'source.test.my_source.my_table', - 'fqn': ['test', 'my_source', 'my_table'], - 'unrendered_config': {}, - }, - }, - 'exposures': { - 'exposure.test.notebook_exposure': { - 'created_at': ANY, - 'depends_on': { - 'macros': [], - 'nodes': ['model.test.model', 'model.test.second_model'] - }, - 'description': 'A description of the complex exposure\n', - 'fqn': ['test', 'notebook_exposure'], - 'maturity': 'medium', - 'meta': {'tool': 'my_tool', 'languages': ['python']}, - 'tags': ['my_department'], - 'name': 'notebook_exposure', - 'original_file_path': self.dir('models/schema.yml'), - 'owner': { - 'email': 'something@example.com', - 'name': 'Some name' - }, - 'package_name': 'test', - 'path': 'schema.yml', - 'refs': [['model'], ['second_model']], - 'resource_type': 'exposure', - 'root_path': self.test_root_realpath, - 'sources': [], - 'type': 'notebook', - 'unique_id': 'exposure.test.notebook_exposure', - 'url': 'http://example.com/notebook/1' - }, - 'exposure.test.simple_exposure': { - 'created_at': ANY, - 'depends_on': { - 'macros': [], - 'nodes': [ - 'source.test.my_source.my_table', - 'model.test.model' - ], - }, - 'description': '', - 'fqn': ['test', 'simple_exposure'], - 'name': 'simple_exposure', - 'original_file_path': self.dir('models/schema.yml'), - 'owner': { - 'email': 'something@example.com', - 'name': None, - }, - 'package_name': 'test', - 'path': 'schema.yml', - 'refs': [['model']], - 'resource_type': 'exposure', - 'root_path': self.test_root_realpath, - 'sources': [['my_source', 'my_table']], - 'type': 'dashboard', - 'unique_id': 'exposure.test.simple_exposure', - 'url': None, - 'maturity': None, - 'meta': {}, - 'tags': [] - } - }, - 'metrics': {}, - 'selectors': {}, - 'parent_map': { - 'model.test.model': ['seed.test.seed'], - 'model.test.second_model': ['seed.test.seed'], - 'exposure.test.notebook_exposure': ['model.test.model', 'model.test.second_model'], - 'exposure.test.simple_exposure': ['model.test.model', 'source.test.my_source.my_table'], - 'seed.test.seed': [], - 'snapshot.test.snapshot_seed': ['seed.test.seed'], - 'source.test.my_source.my_table': [], - 'test.test.not_null_model_id.d01cc630e6': ['model.test.model'], - 'test.test.test_nothing_model_.5d38568946': ['model.test.model'], - 'test.test.unique_model_id.67b76558ff': ['model.test.model'], - }, - 'child_map': { - 'model.test.model': [ - 'exposure.test.notebook_exposure', - 'exposure.test.simple_exposure', - 'test.test.not_null_model_id.d01cc630e6', - 'test.test.test_nothing_model_.5d38568946', - 'test.test.unique_model_id.67b76558ff', - ], - 'model.test.second_model': ['exposure.test.notebook_exposure'], - 'exposure.test.notebook_exposure': [], - 'exposure.test.simple_exposure': [], - 'seed.test.seed': ['model.test.model', - 'model.test.second_model', - 'snapshot.test.snapshot_seed'], - 'snapshot.test.snapshot_seed': [], - 'source.test.my_source.my_table': ['exposure.test.simple_exposure'], - 'test.test.not_null_model_id.d01cc630e6': [], - 'test.test.test_nothing_model_.5d38568946': [], - 'test.test.unique_model_id.67b76558ff': [], - }, - 'docs': { - 'dbt.__overview__': ANY, - 'test.macro_info': ANY, - 'test.macro_arg_info': ANY, - }, - 'disabled': {}, - } - - def expected_bigquery_complex_manifest(self): - nested_view_sql_path = self.dir('bq_models/nested_view.sql') - nested_table_sql_path = self.dir('bq_models/nested_table.sql') - clustered_sql_path = self.dir('bq_models/clustered.sql') - multi_clustered_sql_path = self.dir('bq_models/multi_clustered.sql') - seed_path = self.dir('seed/seed.csv') - snapshot_path = self.dir('snapshot/snapshot_seed.sql') - my_schema_name = self.unique_schema() - - return { - 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v5.json', - 'dbt_version': dbt.version.__version__, - 'nodes': { - 'model.test.clustered': { - 'alias': 'clustered', - 'config': self.rendered_model_config( - cluster_by=['first_name'], - materialized='table', - partition_by={'field': 'updated_at', - 'data_type': 'date'}, - ), - 'sources': [], - 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, - 'fqn': ['test', 'clustered'], - 'compiled_path': Normalized('target/compiled/test/bq_models/clustered.sql'), - 'build_path': None, - 'created_at': ANY, - 'name': 'clustered', - 'original_file_path': clustered_sql_path, - 'package_name': 'test', - 'path': 'clustered.sql', - 'raw_sql': LineIndifferent(_read_file(clustered_sql_path).rstrip('\r\n')), - 'refs': [['seed']], - 'relation_name': '`{0}`.`{1}`.clustered'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.clustered', - 'columns': { - 'email': { - 'description': "The user's email", - 'name': 'email', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'description': "The user's name", - 'name': 'first_name', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'id': { - 'description': 'The user id', - 'name': 'id', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'description': "The user's IP address", - 'name': 'ip_address', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'description': 'When the user was updated', - 'name': 'updated_at', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'deferred': False, - 'description': 'A clustered and partitioned copy of the test model', - 'patch_path': 'test://' + self.dir('bq_models/schema.yml'), - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(clustered_sql_path), - 'unrendered_config': self.unrendered_model_config( - cluster_by=['first_name'], - materialized='table', - partition_by={'field': 'updated_at', - 'data_type': 'date'}, - ), - }, - 'model.test.multi_clustered': { - 'alias': 'multi_clustered', - 'compiled_path': Normalized('target/compiled/test/bq_models/multi_clustered.sql'), - 'build_path': None, - 'created_at': ANY, - 'config': self.rendered_model_config( - cluster_by=['first_name', 'email'], - materialized='table', - partition_by={'field': 'updated_at', - 'data_type': 'date'} - ), - 'sources': [], - 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, - 'fqn': ['test', 'multi_clustered'], - 'name': 'multi_clustered', - 'original_file_path': multi_clustered_sql_path, - 'package_name': 'test', - 'path': 'multi_clustered.sql', - 'raw_sql': LineIndifferent(_read_file(multi_clustered_sql_path).rstrip('\r\n')), - 'refs': [['seed']], - 'relation_name': '`{0}`.`{1}`.multi_clustered'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.multi_clustered', - 'columns': { - 'email': { - 'description': "The user's email", - 'name': 'email', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'description': "The user's name", - 'name': 'first_name', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'id': { - 'description': 'The user id', - 'name': 'id', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'description': "The user's IP address", - 'name': 'ip_address', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'description': 'When the user was updated', - 'name': 'updated_at', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'deferred': False, - 'description': 'A clustered and partitioned copy of the test model, clustered on multiple columns', - 'patch_path': 'test://' + self.dir('bq_models/schema.yml'), - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(multi_clustered_sql_path), - 'unrendered_config': self.unrendered_model_config( - cluster_by=['first_name', 'email'], - materialized='table', - partition_by={'field': 'updated_at', - 'data_type': 'date'} - ), - }, - 'model.test.nested_view': { - 'alias': 'nested_view', - 'compiled_path': Normalized('target/compiled/test/bq_models/nested_view.sql'), - 'build_path': None, - 'created_at': ANY, - 'config': self.rendered_model_config(), - 'sources': [], - 'depends_on': { - 'macros': [], - 'nodes': ['model.test.nested_table'] - }, - 'fqn': ['test', 'nested_view'], - 'name': 'nested_view', - 'original_file_path': nested_view_sql_path, - 'package_name': 'test', - 'path': 'nested_view.sql', - 'raw_sql': LineIndifferent(_read_file(nested_view_sql_path).rstrip('\r\n')), - 'refs': [['nested_table']], - 'relation_name': '`{0}`.`{1}`.nested_view'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.nested_view', - 'columns': { - 'field_1': { - 'name': 'field_1', - 'description': 'The first field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'field_2': { - 'name': 'field_2', - 'description': 'The second field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'field_3': { - 'name': 'field_3', - 'description': 'The third field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'nested_field.field_4': { - 'name': 'nested_field.field_4', - 'description': 'The first nested field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'nested_field.field_5': { - 'name': 'nested_field.field_5', - 'description': 'The second nested field', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'deferred': False, - 'description': 'The test model', - 'patch_path': 'test://' + self.dir('bq_models/schema.yml'), - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(nested_view_sql_path), - 'unrendered_config': self.unrendered_model_config(materialized='view'), - }, - 'model.test.nested_table': { - 'alias': 'nested_table', - 'compiled_path': Normalized('target/compiled/test/bq_models/nested_table.sql'), - 'build_path': None, - 'created_at': ANY, - 'config': self.rendered_model_config(materialized='table'), - 'sources': [], - 'depends_on': { - 'macros': [], - 'nodes': [] - }, - 'fqn': ['test', 'nested_table'], - 'name': 'nested_table', - 'original_file_path': nested_table_sql_path, - 'package_name': 'test', - 'patch_path': None, - 'path': 'nested_table.sql', - 'raw_sql': LineIndifferent(_read_file(nested_table_sql_path).rstrip('\r\n')), - 'refs': [], - 'relation_name': '`{0}`.`{1}`.nested_table'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.nested_table', - 'columns': {}, - 'deferred': False, - 'description': '', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(nested_table_sql_path), - 'unrendered_config': self.unrendered_model_config(materialized='table'), - }, - 'seed.test.seed': { - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'patch_path': 'test://' + self.dir('seed/schema.yml'), - 'path': 'seed.csv', - 'name': 'seed', - 'root_path': self.test_root_realpath, - 'relation_name': '`{0}`.`{1}`.seed'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'seed', - 'raw_sql': '', - 'package_name': 'test', - 'original_file_path': seed_path, - 'refs': [], - 'sources': [], - 'depends_on': { - 'nodes': [], - 'macros': [], - }, - 'unique_id': 'seed.test.seed', - 'fqn': ['test', 'seed'], - 'tags': [], - 'meta': {}, - 'config': self.rendered_seed_config(), - 'schema': my_schema_name, - 'database': self.default_database, - 'alias': 'seed', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'deferred': False, - 'description': 'The test seed', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': '', - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(seed_path), - 'unrendered_config': self.unrendered_seed_config(), - }, - 'snapshot.test.snapshot_seed': { - 'alias': 'snapshot_seed', - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'checksum': self._checksum_file(snapshot_path), - 'columns': {}, - 'compiled': True, - 'compiled_sql': ANY, - 'config': self.rendered_snapshot_config( - target_schema=self.alternate_schema - ), - 'database': self.default_database, - 'deferred': False, - 'depends_on': {'macros': [], - 'nodes': ['seed.test.seed']}, - 'description': '', - 'docs': {'show': True}, - 'extra_ctes': [], - 'extra_ctes_injected': True, - 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], - 'meta': {}, - 'name': 'snapshot_seed', - 'original_file_path': snapshot_path, - 'package_name': 'test', - 'patch_path': None, - 'path': 'snapshot_seed.sql', - 'raw_sql': ANY, - 'refs': [['seed']], - 'relation_name': '`{0}`.`{1}`.snapshot_seed'.format( - self.default_database, self.alternate_schema - ), - 'resource_type': 'snapshot', - 'root_path': self.test_root_realpath, - 'schema': self.alternate_schema, - 'sources': [], - 'tags': [], - 'unique_id': 'snapshot.test.snapshot_seed', - 'unrendered_config': self.unrendered_snapshot_config( - target_schema=self.alternate_schema - )} - }, - 'sources': {}, - 'exposures': {}, - 'metrics': {}, - 'selectors': {}, - 'child_map': { - 'model.test.clustered': [], - 'model.test.multi_clustered': [], - 'model.test.nested_table': ['model.test.nested_view'], - 'model.test.nested_view': [], - 'seed.test.seed': ['model.test.clustered', - 'model.test.multi_clustered', - 'snapshot.test.snapshot_seed'], - 'snapshot.test.snapshot_seed': [] - }, - 'parent_map': { - 'model.test.clustered': ['seed.test.seed'], - 'model.test.multi_clustered': ['seed.test.seed'], - 'seed.test.seed': [], - 'snapshot.test.snapshot_seed': ['seed.test.seed'], - 'model.test.nested_table': [], - 'model.test.nested_view': ['model.test.nested_table'], - }, - 'docs': { - 'dbt.__overview__': ANY, - 'test.macro_info': ANY, - 'test.macro_arg_info': ANY, - }, - 'disabled': {}, - } - - def _checksum_file(self, path): - """windows has silly git behavior that adds newlines, and python does - silly things if we just open(..., 'r').encode('utf-8'). - """ - with open(self.dir(path), 'rb') as fp: - hashed = hashlib.sha256(fp.read()).hexdigest() - return { - 'name': 'sha256', - 'checksum': hashed, - } - - def _path_to(self, searched_path: str, relative_path: str): - return { - 'searched_path': normalize(searched_path), - 'relative_path': normalize(relative_path), - 'project_root': normalize(self.test_root_dir), - } - - def _absolute_path_to(self, searched_path: str, relative_path: str): - return os.path.join( - normalize(self.test_root_dir), - normalize(searched_path), - normalize(relative_path) - ) - - def _relation_name_format(self, quote_database: bool, quote_schema: bool, - quote_identifier: bool): - return ".".join(( - self._quote("{0}") if quote_database else '{0}', - self._quote("{1}") if quote_schema else '{1}', - self._quote("{2}") if quote_identifier else '{2}', - )) - - def verify_metadata(self, metadata, dbt_schema_version): - assert 'generated_at' in metadata - self.assertBetween(metadata['generated_at'], - start=self.generate_start_time) - assert 'dbt_version' in metadata - assert metadata['dbt_version'] == dbt.version.__version__ - assert 'dbt_schema_version' in metadata - assert metadata['dbt_schema_version'] == dbt_schema_version - assert metadata['invocation_id'] == dbt.tracking.active_user.invocation_id - key = 'env_key' - if os.name == 'nt': - key = key.upper() - assert metadata['env'] == { - key: 'env_value' - } - - def verify_manifest(self, expected_manifest): - self.assertTrue(os.path.exists('./target/manifest.json')) - - manifest = _read_json('./target/manifest.json') - - manifest_keys = frozenset({ - 'nodes', 'sources', 'macros', 'parent_map', 'child_map', - 'docs', 'metadata', 'docs', 'disabled', 'exposures', 'selectors', - 'metrics', - }) - - self.assertEqual(frozenset(manifest), manifest_keys) - - for key in manifest_keys: - if key == 'macros': - self.verify_manifest_macros( - manifest, expected_manifest.get('macros')) - elif key == 'metadata': - metadata = manifest['metadata'] - self.verify_metadata( - metadata, 'https://schemas.getdbt.com/dbt/manifest/v5.json') - assert 'project_id' in metadata and metadata[ - 'project_id'] == '098f6bcd4621d373cade4e832627b4f6' - assert 'send_anonymous_usage_stats' in metadata and metadata[ - 'send_anonymous_usage_stats'] is False - assert 'adapter_type' in metadata and metadata['adapter_type'] == self.adapter_type - else: - self.assertIn(key, expected_manifest) # sanity check - self.assertEqual(manifest[key], expected_manifest[key]) - - def _quote(self, value): - quote_char = '`' - return '{0}{1}{0}'.format(quote_char, value) - - def expected_run_results(self): - """ - The expected results of this run. - """ - - return [ - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'model.test.model', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'model.test.second_model', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'seed.test.seed', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'snapshot.test.snapshot_seed', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'test.test.not_null_model_id.d01cc630e6', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'test.test.test_nothing_model_.5d38568946', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'test.test.unique_model_id.67b76558ff', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - ] - - def verify_run_results(self, expected_run_results): - run_results = _read_json('./target/run_results.json') - assert 'metadata' in run_results - self.verify_metadata( - run_results['metadata'], 'https://schemas.getdbt.com/dbt/run-results/v4.json') - self.assertIn('elapsed_time', run_results) - self.assertGreater(run_results['elapsed_time'], 0) - self.assertTrue( - isinstance(run_results['elapsed_time'], float), - "run_results['elapsed_time'] is of type {}, expected float".format( - str(type(run_results['elapsed_time']))) - ) - - assert 'args' in run_results - # sort the results so we can make reasonable assertions - run_results['results'].sort(key=lambda r: r['unique_id']) - assert run_results['results'] == expected_run_results - set(run_results) == {'elapsed_time', 'results', 'metadata'} - - @use_profile('bigquery') - def test__bigquery__run_and_generate(self): - self.run_and_generate() - - self.verify_catalog(self.expected_bigquery_catalog()) - self.verify_manifest(self.expected_seeded_manifest()) - self.verify_run_results(self.expected_run_results()) - - @use_profile('bigquery') - def test__bigquery__complex_models(self): - self.run_and_generate( - extra={'model-paths': [self.dir('bq_models')]}, - model_count=4 - ) - - self.verify_catalog(self.expected_bigquery_complex_catalog()) - self.verify_manifest(self.expected_bigquery_complex_manifest()) - - -class TestDocsGenerateMissingSchema(DBTIntegrationTest): - @property - def schema(self): - return 'docs_generate' - - @staticmethod - def dir(path): - return normalize(path) - - @property - def models(self): - return self.dir("bq_models_noschema") - - def setUp(self): - super().setUp() - self.extra_schema = self.unique_schema() + '_bq_test' - - def tearDown(self): - with self.adapter.connection_named('__test'): - self._drop_schema_named(self.default_database, self.extra_schema) - super().tearDown() - - @use_profile('bigquery') - def test_bigquery_docs_generate_noschema(self): - self.run_dbt([ - 'docs', 'generate', - '--vars', "{{extra_schema: {}}}".format(self.extra_schema) - ]) diff --git a/tests/integration/docs_generate_tests/trivial_models/model.sql b/tests/integration/docs_generate_tests/trivial_models/model.sql deleted file mode 100644 index 43258a714..000000000 --- a/tests/integration/docs_generate_tests/trivial_models/model.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id diff --git a/tox.ini b/tox.ini index 348be15af..80191db8c 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ skip_install = true passenv = DBT_* BIGQUERY_TEST_* PYTEST_ADDOPTS commands = bigquery: {envpython} -m pytest {posargs} -m profile_bigquery tests/integration - bigquery: {envpython} -m pytest {posargs} tests/functional --profile service_account + bigquery: {envpython} -m pytest {posargs} -vv tests/functional --profile service_account deps = -rdev-requirements.txt -e. From 799790e149b980466d966b52a42c33eec4285ef2 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Wed, 1 Jun 2022 11:16:19 -0500 Subject: [PATCH 5/6] Add regression test case (#188) * init push or pytest conversion of override_database for bigquery * tests are currently passing, need to implement some test changes overs (commented out tests) * implemented a vew versions of the check_realations_equal test on test_bigquery_database_override test working on getting the rest to work for first test * trying to grab local env to test against need to find way to grab information from configs * attempting to move over to using check_relations_equal_with_relations * ordering of models * post pair update * debugged test, had passing locally * adding change to conftest to see if it passes tests in ci/cd * removing conftest change * trying ci/cd after some changes * trying to take into account CT-604, remove old test, remove BIGQUERY_TEST_DATABASE env * moving override_database out of adapter folder up one * adding regression test case draft * changing filename to match yml, changes tests -> macros to define path --- .../test_get_columns_incomplete_database.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/functional/test_get_columns_incomplete_database.py diff --git a/tests/functional/test_get_columns_incomplete_database.py b/tests/functional/test_get_columns_incomplete_database.py new file mode 100644 index 000000000..35e37779d --- /dev/null +++ b/tests/functional/test_get_columns_incomplete_database.py @@ -0,0 +1,54 @@ +import pytest +from dbt.tests.util import run_dbt + +# This is to test a edge case found in https://github.com/dbt-labs/dbt-bigquery/pull/165/files + +tests__get_cols_in_sql = """ +{% test get_cols_in(model) %} + + {# The step which causes the issue #} + {%- set relation = api.Relation.create(identifier=model.table) if execute -%} + + {% set columns = adapter.get_columns_in_relation(relation) %} + + select + {% for col in columns %} + {{ col.name }} {{ "," if not loop.last }} + {% endfor %} + + from {{ model }} + limit 0 + +{% endtest %} +""" + +models__my_model = """select 1 as id, 'text' as another_col +""" + +properties__model_yml = """ +version: 2 +models: + - name: my_model + tests: + - get_cols_in +""" + +class TestIncompleteRelationSetup: + @pytest.fixture(scope="class") + def properties(self): + return {"properties__model_yml.yml": properties__model_yml} + + @pytest.fixture(scope="class") + def macros(self): + return {"get_col_in.sql": tests__get_cols_in_sql} + + @pytest.fixture(scope="class") + def models(self): + return { "my_model.sql": models__my_model } + + +class TestIncompleteRelation(TestIncompleteRelationSetup): + + def test_incomplete_relation(self, project): + run_dbt(["build"]) + From 1b85fc50287ac9afd81fd92e715dffa5f0757379 Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Wed, 1 Jun 2022 13:55:02 -0400 Subject: [PATCH 6/6] Fixing pip update for CI (#196) --- .github/workflows/integration.yml | 8 ++++---- .github/workflows/main.yml | 30 +++++++++++++++--------------- .github/workflows/version-bump.yml | 4 ++-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2097fded1..439ed1820 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -153,14 +153,14 @@ jobs: - name: Install python dependencies run: | - pip install --user --upgrade pip - pip install tox - pip --version + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version tox --version - name: Install dbt-core latest run: | - pip install "git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core" + python -m pip install "git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core" - name: Run tox (bigquery) if: matrix.adapter == 'bigquery' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e961dfa7..9e6ef8688 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,9 +52,9 @@ jobs: - name: Install python dependencies run: | - pip install --user --upgrade pip - pip install -r dev-requirements.txt - pip --version + python -m pip install --user --upgrade pip + python -m pip install -r dev-requirements.txt + python -m pip --version pre-commit --version mypy --version dbt --version @@ -88,13 +88,13 @@ jobs: - name: Install python dependencies run: | - pip install --user --upgrade pip - pip install tox - pip --version + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version tox --version - name: Install dbt-core latest run: | - pip install "git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core" + python -m pip install "git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core" - name: Run tox run: tox @@ -130,9 +130,9 @@ jobs: - name: Install python dependencies run: | - pip install --user --upgrade pip - pip install --upgrade setuptools wheel twine check-wheel-contents - pip --version + python -m pip install --user --upgrade pip + python -m pip install --upgrade setuptools wheel twine check-wheel-contents + python -m pip --version - name: Build distributions run: ./scripts/build-dist.sh @@ -183,9 +183,9 @@ jobs: - name: Install python dependencies run: | - pip install --user --upgrade pip - pip install --upgrade wheel - pip --version + python -m pip install --user --upgrade pip + python -m pip install --upgrade wheel + python -m pip --version - uses: actions/download-artifact@v2 with: name: dist @@ -196,13 +196,13 @@ jobs: - name: Install wheel distributions run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/ + find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - name: Check wheel distributions run: | dbt --version - name: Install source distributions run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/ + find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - name: Check source distributions run: | dbt --version diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index dbd36786a..d9df91c52 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -57,7 +57,7 @@ jobs: run: | python3 -m venv env source env/bin/activate - pip install --upgrade pip + python -m pip install --upgrade pip - name: Create PR branch if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }} @@ -69,7 +69,7 @@ jobs: - name: Bumping version run: | source env/bin/activate - pip install -r dev-requirements.txt + python -m pip install -r dev-requirements.txt env/bin/bumpversion --allow-dirty --new-version ${{steps.variables.outputs.VERSION_NUMBER}} major git status