New functionality - Retrieve the description for identical column nam…

…es from upstream models (#61)

* Update gitignore for latest version of dbt

* Add macros to retrieve info from graph

* Update README with new parameter  for generate_model_yaml

* Add integration test for the new feature

* Update CHANGELOG with upstream_descriptions flag

* Fix typos

* Update CI steps to run the models before testing

.circleci/config.yml

@@ -44,6 +44,7 @@ jobs:
             dbt --warn-error deps --target postgres
             dbt --warn-error run-operation create_source_table --target postgres
             dbt --warn-error seed --target postgres --full-refresh
+            dbt --warn-error run --target postgres
             dbt --warn-error test --target postgres
 
       - run:
@@ -55,6 +56,7 @@ jobs:
             dbt --warn-error deps --target redshift
             dbt --warn-error run-operation create_source_table --target redshift
             dbt --warn-error seed --target redshift --full-refresh
+            dbt --warn-error run --target redshift
             dbt --warn-error test --target redshift
 
       - run:
@@ -66,6 +68,7 @@ jobs:
             dbt --warn-error deps --target snowflake
             dbt --warn-error run-operation create_source_table --target snowflake
             dbt --warn-error seed --target snowflake --full-refresh
+            dbt --warn-error run --target snowflake
             dbt --warn-error test --target snowflake
 
       - run:
@@ -80,6 +83,7 @@ jobs:
             dbt --warn-error deps --target bigquery
             dbt --warn-error run-operation create_source_table --target bigquery
             dbt --warn-error seed --target bigquery --full-refresh
+            dbt --warn-error run --target bigquery
             dbt --warn-error test --target bigquery
 
 

.gitignore

@@ -1,4 +1,5 @@
 
 target/
 dbt_modules/
+dbt_packages/
 logs/

CHANGELOG.md

@@ -1,3 +1,7 @@
+# Unreleased
+## New features
+- Add support for importing descriptions from columns with the same names in upstream models. It is available by setting the parameter `upstream_descriptions` to `True` in `generate_model_yaml` ([#61](https://github.com/dbt-labs/dbt-codegen/pull/61))
+
 # dbt-codegen v0.6.0
 
 This release creates breaking changes to the `generate_source.sql` macro.

README.md

@@ -128,6 +128,7 @@ schema.yml file.
 
 ### Arguments:
 * `model_name` (required): The model you wish to generate YAML for.
+* `upstream_descriptions` (optional, default=False): Whether you want to include descriptions for identical column names from upstream models.
 
 ### Usage:
 1. Create a model.

integration_tests/models/child_model.sql

@@ -0,0 +1,3 @@
+select 
+    * 
+from {{ ref('model_data_a') }}

integration_tests/models/model_data_a.sql

@@ -0,0 +1,3 @@
+select 
+    * 
+from {{ ref('data__a_relation') }}

integration_tests/models/schema.yml

@@ -0,0 +1,7 @@
+version: 2
+
+models:
+  - name: model_data_a
+    columns:
+      - name: col_a
+        description: description column a

integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql

@@ -0,0 +1,22 @@
+{% set actual_model_yaml = codegen.generate_model_yaml(
+    model_name='child_model',
+    upstream_descriptions=True
+  )
+%}
+
+{% set expected_model_yaml %}
+version: 2
+
+models:
+  - name: child_model
+    description: ""
+    columns:
+      - name: col_a
+        description: "description column a"
+
+      - name: col_b
+        description: ""
+
+{% endset %}
+
+{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }}

macros/generate_model_yaml.sql

@@ -1,6 +1,7 @@
-{% macro generate_model_yaml(model_name) %}
+{% macro generate_model_yaml(model_name, upstream_descriptions=False) %}
 
 {% set model_yaml=[] %}
+{% set column_desc_dict =  codegen.build_dict_column_descriptions(model_name) if upstream_descriptions else {} %}
 
 {% do model_yaml.append('version: 2') %}
 {% do model_yaml.append('') %}
@@ -14,7 +15,7 @@
 
 {% for column in columns %}
     {% do model_yaml.append('      - name: ' ~ column.name | lower ) %}
-    {% do model_yaml.append('        description: ""') %}
+    {% do model_yaml.append('        description: "' ~ column_desc_dict.get(column.name | lower,'') ~ '"') %}
     {% do model_yaml.append('') %}
 {% endfor %}
 

macros/helpers/helpers.sql

@@ -0,0 +1,29 @@
+{# retrieve models directly upstream from a given model #}
+{% macro get_model_dependencies(model_name) %}
+    {% for node in graph.nodes.values() | selectattr('name', "equalto", model_name) %}
+        {{ return(node.depends_on.nodes) }}
+    {% endfor %}
+{% endmacro %}
+
+
+{# add to an input dictionary entries containing all the column descriptions of a given model #}
+{% macro add_model_column_descriptions_to_dict(model_name,dict_with_descriptions={}) %}
+    {% for node in graph.nodes.values() | selectattr('name', "equalto", model_name) %}
+        {% for col_name, col_values in node.columns.items() %}
+            {% do dict_with_descriptions.update( {col_name: col_values.description} ) %}
+        {% endfor %}
+    {% endfor %}
+    {{ return(dict_with_descriptions) }}
+{% endmacro %}
+
+{# build a global dictionary looping through all the direct parents models #}
+{# if the same column name exists with different descriptions it is overwritten at each loop #}
+{% macro build_dict_column_descriptions(model_name) %}
+    {% if execute %}
+        {% set glob_dict = {} %}
+        {% for full_model in codegen.get_model_dependencies(model_name) %}
+            {% do codegen.add_model_column_descriptions_to_dict(full_model.split('.')[-1],glob_dict) %}
+        {% endfor %}
+        {{ return(glob_dict) }}
+    {% endif %}
+{% endmacro %}