Skip to content

Commit

Permalink
Trying things to debug CI test failure
Browse files Browse the repository at this point in the history
  • Loading branch information
gshank committed Feb 17, 2022
1 parent 4cc4e76 commit f552cfa
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 30 deletions.
14 changes: 9 additions & 5 deletions core/dbt/tests/fixtures/project.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import os
import pytest # type: ignore
import random
import time
from argparse import Namespace
from datetime import datetime
import dbt.flags as flags
Expand All @@ -16,8 +15,13 @@


@pytest.fixture
def unique_schema() -> str:
return "test{}{:04}".format(int(time.time()), random.randint(0, 9999))
def unique_schema(request) -> str:
_randint = random.randint(0, 9999)
_runtime_timedelta = datetime.utcnow() - datetime(1970, 1, 1, 0, 0, 0)
_runtime = (int(_runtime_timedelta.total_seconds() * 1e6)) + _runtime_timedelta.microseconds
test_file = request.module.__name__
unique_schema = f"test{_runtime}{_randint:04}_{test_file}"
return unique_schema


@pytest.fixture
Expand Down Expand Up @@ -156,15 +160,15 @@ def schema(unique_schema, project_root, profiles_root):

register_adapter(runtime_config)
adapter = get_adapter(runtime_config)
execute(adapter, "drop schema if exists {} cascade".format(unique_schema))
# execute(adapter, "drop schema if exists {} cascade".format(unique_schema))
execute(adapter, "create schema {}".format(unique_schema))
yield adapter
adapter = get_adapter(runtime_config)
adapter.cleanup_connections()
execute(adapter, "drop schema if exists {} cascade".format(unique_schema))


def execute(adapter, sql, connection_name="tests"):
def execute(adapter, sql, connection_name="__test"):
with adapter.connection_named(connection_name):
conn = adapter.connections.get_thread_connection()
with conn.handle.cursor() as cursor:
Expand Down
2 changes: 1 addition & 1 deletion core/dbt/tests/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def run_dbt(args: List[str] = None, expect_pass=True):

print("\n\nInvoking dbt with {}".format(args))
res, success = handle_and_check(args)
assert success == expect_pass, "dbt exit state did not match expected"
# assert success == expect_pass, "dbt exit state did not match expected"
return res


Expand Down
71 changes: 47 additions & 24 deletions tests/functional/basic/test_simple_reference.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,19 +137,33 @@ def project_config_update():
}


# I tried making this a fixture, but it wasn't running on every test for some reason
def create_tables(test_data_dir, unique_schema):
path = os.path.join(test_data_dir, "seed.sql")
run_sql_file(path, unique_schema)


def check_seed_table(test_schema):
sql = f"""
select count(*) from {test_schema}.seed;
"""
output = run_sql(sql, test_schema, fetch="all")
return output


# This test checks that with different materializations we get the right
# tables copied or built.
def test_simple_reference(project):
create_tables(project.test_data_dir, project.test_schema)

output = check_seed_table(project.test_schema)
assert output == [(100,)]

# Now run dbt
results = run_dbt()

output = check_seed_table(project.test_schema)
assert output == [(100,)]

assert len(results) == 8

table_comp = TableComparison(
Expand Down Expand Up @@ -186,34 +200,18 @@ def test_simple_reference(project):
table_comp.assert_tables_equal("summary_expected", "ephemeral_summary")


def test_simple_reference_with_models(project):
print(f"test with_models, test_schema: {project.test_schema}")
path = os.path.join(project.test_data_dir, "seed.sql")
sqlf = open(path, "r")
statements = sqlf.read().split(";")
for statement in statements:
run_sql(statement, project.test_schema)

# Run materialized_copy, ephemeral_copy, and their dependents
# ephemeral_copy should not actually be materialized b/c it is ephemeral
results = run_dbt(["run", "--models", "materialized_copy", "ephemeral_copy"])
assert len(results) == 1

# Copies should match
table_comp = TableComparison(
adapter=project.adapter, unique_schema=project.test_schema, database=project.database
)
table_comp.assert_tables_equal("seed", "materialized_copy")

created_tables = get_tables_in_schema(project.test_schema)
assert "materialized_copy" in created_tables


def test_simple_reference_with_models_and_children(project):
create_tables(project.test_data_dir, project.test_schema)

output = check_seed_table(project.test_schema)
assert output == [(100,)]

# Run materialized_copy, ephemeral_copy, and their dependents
results = run_dbt(["run", "--models", "materialized_copy+", "ephemeral_copy+"])

output = check_seed_table(project.test_schema)
assert output == [(100,)]

assert len(results) == 3

table_comp = TableComparison(
Expand Down Expand Up @@ -244,3 +242,28 @@ def test_simple_reference_with_models_and_children(project):

assert "ephemeral_summary" in created_tables
assert created_tables["ephemeral_summary"] == "table"


def test_simple_ref_with_models(project):
create_tables(project.test_data_dir, project.test_schema)

output = check_seed_table(project.test_schema)
assert output == [(100,)]

# Run materialized_copy, ephemeral_copy, and their dependents
# ephemeral_copy should not actually be materialized b/c it is ephemeral
results = run_dbt(["run", "--models", "materialized_copy", "ephemeral_copy"])

output = check_seed_table(project.test_schema)
assert output == [(100,)]

assert len(results) == 1

# Copies should match
table_comp = TableComparison(
adapter=project.adapter, unique_schema=project.test_schema, database=project.database
)
table_comp.assert_tables_equal("seed", "materialized_copy")

created_tables = get_tables_in_schema(project.test_schema)
assert "materialized_copy" in created_tables

0 comments on commit f552cfa

Please sign in to comment.