Feature/tables option for codegen

CHANGELOG.md

@@ -1,4 +1,11 @@
-# dbt-codegen 0.5.0
+# dbt-codegen v0.6.0
+
+This release creates breaking changes to the `generate_source.sql` macro.
+
+## Features
+- add optional `table_pattern` argument to `generate_source.sql` macro. Default value is '%' to pull all tables in the raw data schema to preserve existing behavior if the `table_pattern` argument is not specified by the user.
+
+# dbt-codegen v0.5.0
 
 This release supports any version (minor and patch) of v1, which means far less need for compatibility releases in the future.
 

README.md

@@ -25,6 +25,9 @@ source data is in.
 column names to your source definition.
 * `include_descriptions` (optional, default=False): Whether you want to add 
 description placeholders to your source definition.
+* `table_pattern` (optional, default='%'): A table prefix / postfix that you 
+want to subselect from all available tables within a given schema.
+* `exclude` (optional, default=''): A string you want to exclude from the selection criteria
 
 ### Usage:
 1. Copy the macro into a statement tab in the dbt Cloud IDE, or into an analysis file, and compile your code

integration_tests/seeds/data__b_relation.csv

@@ -0,0 +1,3 @@
+col_a,col_b
+3,c
+4,d

integration_tests/tests/test_generate_source.sql

@@ -11,6 +11,7 @@ sources:
   - name: {{ raw_schema | trim }}
     tables:
       - name: data__a_relation
+      - name: data__b_relation
 {% endset %}
 
 

integration_tests/tests/test_generate_source_all_args.sql

@@ -3,13 +3,16 @@
 -- test all args
 {% set actual_source_yaml = codegen.generate_source(
     schema_name=raw_schema,
+    table_pattern='%',
+    exclude='',
     database_name=target.database,
     generate_columns=True,
     include_descriptions=True
 ) %}
 
 
 {% set expected_source_yaml %}
+
 version: 2
 
 sources:
@@ -22,6 +25,13 @@ sources:
           - name: col_b
             description: ""
 
+      - name: data__b_relation
+        columns:
+          - name: col_a
+            description: ""
+          - name: col_b
+            description: ""
+
 {% endset %}
 
 {{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}

integration_tests/tests/test_generate_source_exclude.sql

@@ -0,0 +1,17 @@
+
+{% set raw_schema = generate_schema_name('raw_data') %}
+
+-- test default args
+{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__%', exclude='data__a_%') %}
+
+{% set expected_source_yaml %}
+version: 2
+
+sources:
+  - name: {{ raw_schema | trim }}
+    tables:
+      - name: data__b_relation
+{% endset %}
+
+
+{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}

integration_tests/tests/test_generate_source_table_pattern.sql

@@ -0,0 +1,17 @@
+
+{% set raw_schema = generate_schema_name('raw_data') %}
+
+-- test default args
+{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__b_%') %}
+
+{% set expected_source_yaml %}
+version: 2
+
+sources:
+  - name: {{ raw_schema | trim }}
+    tables:
+      - name: data__b_relation
+{% endset %}
+
+
+{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}

macros/generate_source.sql

@@ -1,9 +1,10 @@
-{% macro get_tables_in_schema(schema_name, database_name=target.database) %}
-
+{% macro get_tables_in_schema(schema_name, database_name=target.database, table_pattern='%', exclude='') %}
+    
     {% set tables=dbt_utils.get_relations_by_pattern(
-            database=database_name,
-            schema_pattern=schema_name,
-            table_pattern='%'
+        schema_pattern=schema_name,
+        database=database_name,
+        table_pattern=table_pattern,
+        exclude=exclude
     ) %}
 
     {% set table_list= tables | map(attribute='identifier') %}
@@ -14,7 +15,7 @@
 
 
 ---
-{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False) %}
+{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='%', exclude='') %}
 
 {% set sources_yaml=[] %}
 
@@ -29,7 +30,7 @@
 
 {% do sources_yaml.append('    tables:') %}
 
-{% set tables=codegen.get_tables_in_schema(schema_name, database_name) %}
+{% set tables=codegen.get_tables_in_schema(schema_name, database_name, table_pattern, exclude) %}
 
 {% for table in tables %}
     {% do sources_yaml.append('      - name: ' ~ table | lower ) %}