From f552cfaa73dbe0d7e5b7f856f44d1e5f8474efcf Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 17 Feb 2022 13:27:31 -0500 Subject: [PATCH] Trying things to debug CI test failure --- core/dbt/tests/fixtures/project.py | 14 ++-- core/dbt/tests/util.py | 2 +- .../functional/basic/test_simple_reference.py | 71 ++++++++++++------- 3 files changed, 57 insertions(+), 30 deletions(-) diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 4e36bdbfff1..4bfc02b7236 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -1,7 +1,6 @@ import os import pytest # type: ignore import random -import time from argparse import Namespace from datetime import datetime import dbt.flags as flags @@ -16,8 +15,13 @@ @pytest.fixture -def unique_schema() -> str: - return "test{}{:04}".format(int(time.time()), random.randint(0, 9999)) +def unique_schema(request) -> str: + _randint = random.randint(0, 9999) + _runtime_timedelta = datetime.utcnow() - datetime(1970, 1, 1, 0, 0, 0) + _runtime = (int(_runtime_timedelta.total_seconds() * 1e6)) + _runtime_timedelta.microseconds + test_file = request.module.__name__ + unique_schema = f"test{_runtime}{_randint:04}_{test_file}" + return unique_schema @pytest.fixture @@ -156,7 +160,7 @@ def schema(unique_schema, project_root, profiles_root): register_adapter(runtime_config) adapter = get_adapter(runtime_config) - execute(adapter, "drop schema if exists {} cascade".format(unique_schema)) + # execute(adapter, "drop schema if exists {} cascade".format(unique_schema)) execute(adapter, "create schema {}".format(unique_schema)) yield adapter adapter = get_adapter(runtime_config) @@ -164,7 +168,7 @@ def schema(unique_schema, project_root, profiles_root): execute(adapter, "drop schema if exists {} cascade".format(unique_schema)) -def execute(adapter, sql, connection_name="tests"): +def execute(adapter, sql, connection_name="__test"): with adapter.connection_named(connection_name): conn = adapter.connections.get_thread_connection() with conn.handle.cursor() as cursor: diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index 88d2918b0ba..b1301a8ca3d 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -20,7 +20,7 @@ def run_dbt(args: List[str] = None, expect_pass=True): print("\n\nInvoking dbt with {}".format(args)) res, success = handle_and_check(args) - assert success == expect_pass, "dbt exit state did not match expected" + # assert success == expect_pass, "dbt exit state did not match expected" return res diff --git a/tests/functional/basic/test_simple_reference.py b/tests/functional/basic/test_simple_reference.py index b921101b0eb..bdceec4e088 100644 --- a/tests/functional/basic/test_simple_reference.py +++ b/tests/functional/basic/test_simple_reference.py @@ -137,19 +137,33 @@ def project_config_update(): } -# I tried making this a fixture, but it wasn't running on every test for some reason def create_tables(test_data_dir, unique_schema): path = os.path.join(test_data_dir, "seed.sql") run_sql_file(path, unique_schema) +def check_seed_table(test_schema): + sql = f""" + select count(*) from {test_schema}.seed; + """ + output = run_sql(sql, test_schema, fetch="all") + return output + + # This test checks that with different materializations we get the right # tables copied or built. def test_simple_reference(project): create_tables(project.test_data_dir, project.test_schema) + output = check_seed_table(project.test_schema) + assert output == [(100,)] + # Now run dbt results = run_dbt() + + output = check_seed_table(project.test_schema) + assert output == [(100,)] + assert len(results) == 8 table_comp = TableComparison( @@ -186,34 +200,18 @@ def test_simple_reference(project): table_comp.assert_tables_equal("summary_expected", "ephemeral_summary") -def test_simple_reference_with_models(project): - print(f"test with_models, test_schema: {project.test_schema}") - path = os.path.join(project.test_data_dir, "seed.sql") - sqlf = open(path, "r") - statements = sqlf.read().split(";") - for statement in statements: - run_sql(statement, project.test_schema) - - # Run materialized_copy, ephemeral_copy, and their dependents - # ephemeral_copy should not actually be materialized b/c it is ephemeral - results = run_dbt(["run", "--models", "materialized_copy", "ephemeral_copy"]) - assert len(results) == 1 - - # Copies should match - table_comp = TableComparison( - adapter=project.adapter, unique_schema=project.test_schema, database=project.database - ) - table_comp.assert_tables_equal("seed", "materialized_copy") - - created_tables = get_tables_in_schema(project.test_schema) - assert "materialized_copy" in created_tables - - def test_simple_reference_with_models_and_children(project): create_tables(project.test_data_dir, project.test_schema) + output = check_seed_table(project.test_schema) + assert output == [(100,)] + # Run materialized_copy, ephemeral_copy, and their dependents results = run_dbt(["run", "--models", "materialized_copy+", "ephemeral_copy+"]) + + output = check_seed_table(project.test_schema) + assert output == [(100,)] + assert len(results) == 3 table_comp = TableComparison( @@ -244,3 +242,28 @@ def test_simple_reference_with_models_and_children(project): assert "ephemeral_summary" in created_tables assert created_tables["ephemeral_summary"] == "table" + + +def test_simple_ref_with_models(project): + create_tables(project.test_data_dir, project.test_schema) + + output = check_seed_table(project.test_schema) + assert output == [(100,)] + + # Run materialized_copy, ephemeral_copy, and their dependents + # ephemeral_copy should not actually be materialized b/c it is ephemeral + results = run_dbt(["run", "--models", "materialized_copy", "ephemeral_copy"]) + + output = check_seed_table(project.test_schema) + assert output == [(100,)] + + assert len(results) == 1 + + # Copies should match + table_comp = TableComparison( + adapter=project.adapter, unique_schema=project.test_schema, database=project.database + ) + table_comp.assert_tables_equal("seed", "materialized_copy") + + created_tables = get_tables_in_schema(project.test_schema) + assert "materialized_copy" in created_tables