dbt-core v1.0.0

.github/workflows/test.yml

@@ -30,7 +30,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: [3.7, 3.x]
-        dbt-version: [0.18.x, 0.19.x]
+        dbt-version: [0.18.x, 0.19.x, 1.x]
         exclude:
           - python-version: 3.x
             dbt-version: 0.18.x

dbt_invoke/internal/_utils.py

@@ -192,6 +192,7 @@ def get_cli_kwargs(**kwargs):
 def dbt_run_operation(
     ctx,
     macro_name,
+    log_format='json',
     project_dir=None,
     profiles_dir=None,
     profile=None,
@@ -208,6 +209,7 @@ def dbt_run_operation(
 
     :param ctx: An Invoke context object
     :param macro_name: Name of macro that will be run
+    :param log_format: Global cli flag to set the log format, including those send to stdout
     :param project_dir: An argument for the dbt run-operation command
         (run "dbt run-operation --help" for details)
     :param profiles_dir: An argument for the dbt run-operation command
@@ -236,6 +238,12 @@ def dbt_run_operation(
         'bypass_cache': bypass_cache,
     }
     dbt_cli_kwargs = get_cli_kwargs(**dbt_kwargs)
+
+    dbt_global_kwargs = {
+        'log-format': log_format
+    }
+    dbt_global_cli_kwargs = get_cli_kwargs(**dbt_global_kwargs)
+
     macro_kwargs = json.dumps(kwargs, sort_keys=False)
     if platform.system().lower().startswith('win'):
         # Format YAML string for Windows Command Prompt
@@ -249,12 +257,12 @@ def dbt_run_operation(
         macro_kwargs = macro_kwargs.replace("'", """'"'"'""")
         macro_kwargs = f"'{macro_kwargs}'"
     command = (
-        f"dbt run-operation {dbt_cli_kwargs}"
+        f"dbt {dbt_global_cli_kwargs} run-operation {dbt_cli_kwargs}"
         f" {macro_name} --args {macro_kwargs}"
     )
     logger.debug(f'Running command: {command}')
     result = ctx.run(command, hide=hide)
-    result_lines = result.stdout.splitlines()[1:]
+    result_lines = [json.loads(data) for data in result.stdout.splitlines()]
     return result_lines
 
 

dbt_invoke/properties.py

@@ -6,6 +6,7 @@
 from invoke import task
 
 from dbt_invoke.internal import _utils
+import ast
 
 _LOGGER = _utils.get_logger('dbt-invoke')
 _MACRO_NAME = '_log_columns_list'
@@ -502,13 +503,22 @@ def _get_columns(ctx, resource_location, resource_dict, **kwargs):
         result_lines = _utils.dbt_run_operation(
             ctx, _MACRO_NAME, hide=True, logger=_LOGGER, sql=sql, **kwargs
         )
-    result_list_strings = [
-        s for s in result_lines if s.startswith('[') and s.endswith(']')
-    ]
-    # Take the last line that started with '[' and ended with ']'
-    # (just in case there is multi-line output)
-    columns_string = result_list_strings[-1].strip('][').split(', ')
-    columns = [col.strip("'") for col in columns_string]
+
+    # from dbt-core v1.0.0 onwards, run_operations INFO logs have code M011 and messages have key 'msg'
+    # https://github.com/dbt-labs/dbt-core/blob/22b1a09aa218e8152b0c2dd261abe2503ea15ddb/core/dbt/events/types.py#L401
+    relevant_line = list(filter(lambda x: x.get('code') == 'M011', result_lines))
+
+    if len(relevant_line) == 1:
+        columns = relevant_line[0].get('msg')
+
+    else:
+        # for older dbt-core versions, we need to cross fingers a little harder
+        relevant_line = result_lines[1:]
+        # also, the message key is different
+        columns = relevant_line[0].get('message')
+        # columns are not passed as valid json but as a string representation of a list
+        columns = ast.literal_eval(columns)
+
     return columns
 
 

requirements/requirements_dbt_1.x.txt

@@ -0,0 +1,7 @@
+agate<1.6.2
+dbt-core~=1.0.0
+dbt-sqlite~=0.2.2
+invoke>=1.4.1
+pytz<2021.0
+PyYAML>=5.1
+-e .

tests/dbt_project_files/dbt_project.yml

@@ -0,0 +1,18 @@
+name: test_dbt_project
+version: 1.0.0
+config-version: 2
+
+profile: dbt-sqlite
+
+seed-paths: ["data"]
+
+clean-targets:
+  - target
+  - dbt_packages
+  - logs
+
+analysis-paths:
+  - analyses
+
+snapshots:
+  +target_schema: main

tests/dbt_project_files/dbt_project_pre_dbt_v1.yml

@@ -0,0 +1,16 @@
+name: test_dbt_project
+version: 1.0.0
+config-version: 2
+
+profile: dbt-sqlite
+
+clean-targets:
+  - target
+  - dbt_modules
+  - logs
+
+analysis-paths:
+  - analyses
+
+snapshots:
+  +target_schema: main

tests/test.py

@@ -9,6 +9,9 @@
 from dbt_invoke import properties
 from dbt_invoke.internal import _utils
 
+import pkg_resources
+import shutil
+
 PARENT_DIR = Path(__file__).parent
 
 
@@ -23,6 +26,15 @@ def setUpClass(cls):
         cls.logger = _utils.get_logger('dbt-invoke', level='DEBUG')
         cls.config_path = Path(PARENT_DIR, 'test_config.yml')
         cls.config = _utils.parse_yaml(cls.config_path)
+
+        # for backward compatibility, select the correct dbt_project.yml file
+        if pkg_resources.get_distribution("dbt-core").version >= '1.0.0':
+            shutil.copy(Path(PARENT_DIR, 'dbt_project_files/dbt_project.yml'),
+                        Path(PARENT_DIR, cls.config['project_name'], 'dbt_project.yml'))
+        else:
+            shutil.copy(Path(PARENT_DIR, 'dbt_project_files/dbt_project_pre_dbt_v1.yml'),
+                        Path(PARENT_DIR, cls.config['project_name'], 'dbt_project.yml'))
+
         cls.project_dir = Path(PARENT_DIR, cls.config['project_name'])
         cls.profiles_dir = Path(PARENT_DIR, cls.config['project_name'])
         cls.expected_properties = cls.config['expected_properties']