Fix errors on Windows tests in new tests/functional (#4767)

* [#4781] Convert reads and writes in project fixture to text/utf-8 encoding

* Switch to using write_file and read_file functions

* Add comment

CHANGELOG.md

@@ -21,6 +21,7 @@
 - Re-format codebase (except tests) using pre-commit hooks ([#3195](https://github.com/dbt-labs/dbt-core/issues/3195), [#4697](https://github.com/dbt-labs/dbt-core/pull/4697))
 - Add deps module README ([#4686](https://github.com/dbt-labs/dbt-core/pull/4686/))
 - Initial conversion of tests to pytest ([#4690](https://github.com/dbt-labs/dbt-core/issues/4690), [#4691](https://github.com/dbt-labs/dbt-core/pull/4691))
+- Fix errors in Windows for tests/functions ([#4781](https://github.com/dbt-labs/dbt-core/issues/4781), [#4767](https://github.com/dbt-labs/dbt-core/pull/4767))
 
 Contributors:
 - [@NiallRees](https://github.com/NiallRees) ([#4447](https://github.com/dbt-labs/dbt-core/pull/4447))

core/dbt/tests/fixtures/project.py

@@ -14,6 +14,7 @@
 from dbt.context import providers
 from dbt.events.functions import fire_event
 from dbt.events.test_types import IntegrationTestDebug
+from dbt.tests.util import write_file
 
 
 # These are the fixtures that are used in dbt core functional tests
@@ -111,9 +112,7 @@ def dbt_profile_data(unique_schema, database_host):
 @pytest.fixture
 def profiles_yml(profiles_root, dbt_profile_data):
     os.environ["DBT_PROFILES_DIR"] = str(profiles_root)
-    path = os.path.join(profiles_root, "profiles.yml")
-    with open(path, "w") as fp:
-        fp.write(yaml.safe_dump(dbt_profile_data))
+    write_file(yaml.safe_dump(dbt_profile_data), profiles_root, "profiles.yml")
     yield dbt_profile_data
     del os.environ["DBT_PROFILES_DIR"]
 
@@ -137,8 +136,7 @@ def dbt_project_yml(project_root, project_config_update, logs_dir):
     }
     if project_config_update:
         project_config.update(project_config_update)
-    runtime_config_file = project_root.join("dbt_project.yml")
-    runtime_config_file.write(yaml.safe_dump(project_config))
+    write_file(yaml.safe_dump(project_config), project_root, "dbt_project.yml")
 
 
 # Fixture to provide packages as either yaml or dictionary
@@ -155,7 +153,7 @@ def packages_yml(project_root, packages):
             data = packages
         else:
             data = yaml.safe_dump(packages)
-        project_root.join("packages.yml").write(data)
+        write_file(data, project_root, "packages.yml")
 
 
 # Fixture to provide selectors as either yaml or dictionary
@@ -172,7 +170,7 @@ def selectors_yml(project_root, selectors):
             data = selectors
         else:
             data = yaml.safe_dump(selectors)
-        project_root.join("selectors.yml").write(data)
+        write_file(data, project_root, "selectors.yml")
 
 
 # This creates an adapter that is used for running test setup and teardown,
@@ -205,13 +203,13 @@ def write_project_files(project_root, dir_name, file_dict):
 def write_project_files_recursively(path, file_dict):
     for name, value in file_dict.items():
         if name.endswith(".sql") or name.endswith(".csv") or name.endswith(".md"):
-            path.join(name).write(value)
+            write_file(value, path, name)
         elif name.endswith(".yml") or name.endswith(".yaml"):
             if isinstance(value, str):
                 data = value
             else:
                 data = yaml.safe_dump(value)
-            path.join(name).write(data)
+            write_file(data, path, name)
         else:
             write_project_files_recursively(path.mkdir(name), value)
 

core/dbt/tests/util.py

@@ -71,3 +71,17 @@ def copy_file(src_path, src, dest_path, dest) -> None:
 def rm_file(src_path, src) -> None:
     # remove files from proj_path
     os.remove(os.path.join(src_path, src))
+
+
+# We need to explicitly use encoding="utf-8" because otherwise on
+# Windows we'll get codepage 1252 and things might break
+def write_file(contents, *paths):
+    with open(os.path.join(*paths), "w", encoding="utf-8") as fp:
+        fp.write(contents)
+
+
+def read_file(*paths):
+    contents = ""
+    with open(os.path.join(*paths), "r") as fp:
+        contents = fp.read()
+    return contents

tests/fixtures/jaffle_shop.py

@@ -1,5 +1,6 @@
 import pytest
 import os
+from dbt.tests.util import read_file
 
 # models/customers.sql
 customers_sql = """
@@ -394,9 +395,8 @@ def seeds():
     seeds = {}
     dir_path = os.path.dirname(os.path.realpath(__file__))
     for file_name in ("raw_customers.csv", "raw_orders.csv", "raw_payments.csv"):
-        path = os.path.join(dir_path, "jaffle_shop_data", file_name)
-        with open(path, "rb") as fp:
-            seeds[file_name] = fp.read()
+        contents = read_file(dir_path, "jaffle_shop_data", file_name)
+        seeds[file_name] = contents
     return seeds
 
 

tests/functional/basic/test_copy_uppercase.py

@@ -1,7 +1,6 @@
 import pytest
-import os
 from dbt.tests.tables import TableComparison
-from dbt.tests.util import run_dbt
+from dbt.tests.util import run_dbt, read_file
 
 from tests.functional.basic.test_simple_copy import (
     advanced_incremental_sql,
@@ -63,11 +62,8 @@ def models():
 @pytest.fixture
 def seeds(test_data_dir):
     # Read seed file and return
-    path = os.path.join(test_data_dir, "seed-initial.csv")
-    with open(path, "rb") as fp:
-        seed_csv = fp.read()
-        return {"seed.csv": seed_csv}
-    return {}
+    seed_csv = read_file(test_data_dir, "seed-initial.csv")
+    return {"seed.csv": seed_csv}
 
 
 def test_simple_copy_uppercase(project):

tests/functional/basic/test_simple_copy.py

@@ -1,6 +1,5 @@
 import pytest
-import os
-from dbt.tests.util import run_dbt, copy_file
+from dbt.tests.util import run_dbt, copy_file, read_file
 from dbt.tests.tables import TableComparison
 
 # advanced_incremental.sql
@@ -146,11 +145,8 @@ def models():
 @pytest.fixture
 def seeds(test_data_dir):
     # Read seed file and return
-    path = os.path.join(test_data_dir, "seed-initial.csv")
-    with open(path, "rb") as fp:
-        seed_csv = fp.read()
-        return {"seed.csv": seed_csv}
-    return {}
+    seed_csv = read_file(test_data_dir, "seed-initial.csv")
+    return {"seed.csv": seed_csv}
 
 
 @pytest.fixture

tests/functional/basic/test_simple_reference.py

@@ -1,6 +1,5 @@
 import pytest
-import os
-from dbt.tests.util import run_dbt, copy_file
+from dbt.tests.util import run_dbt, copy_file, read_file
 from dbt.tests.tables import TableComparison
 
 
@@ -140,14 +139,10 @@ def models():
 def seeds(test_data_dir):
     # Read seed file and return
     seeds = {"properties.yml": properties_yml}
-    path = os.path.join(test_data_dir, "seed-initial.csv")
-    with open(path, "rb") as fp:
-        seed_csv = fp.read()
-        seeds["users.csv"] = seed_csv
-    path = os.path.join(test_data_dir, "summary_expected.csv")
-    with open(path, "rb") as fp:
-        summary_csv = fp.read()
-        seeds["summary_expected.csv"] = summary_csv
+    seed_csv = read_file(test_data_dir, "seed-initial.csv")
+    seeds["users.csv"] = seed_csv
+    summary_csv = read_file(test_data_dir, "summary_expected.csv")
+    seeds["summary_expected.csv"] = summary_csv
     return seeds