diff --git a/CHANGELOG.md b/CHANGELOG.md index 31f148f..53f6c97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ -# dbt-codegen 0.5.0 +# dbt-codegen v0.6.0 + +This release creates breaking changes to the `generate_source.sql` macro. + +## Features +- add optional `table_pattern` argument to `generate_source.sql` macro. Default value is '%' to pull all tables in the raw data schema to preserve existing behavior if the `table_pattern` argument is not specified by the user. + +# dbt-codegen v0.5.0 This release supports any version (minor and patch) of v1, which means far less need for compatibility releases in the future. diff --git a/README.md b/README.md index 7be3bc8..d53f75d 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ source data is in. column names to your source definition. * `include_descriptions` (optional, default=False): Whether you want to add description placeholders to your source definition. +* `table_pattern` (optional, default='%'): A table prefix / postfix that you +want to subselect from all available tables within a given schema. +* `exclude` (optional, default=''): A string you want to exclude from the selection criteria ### Usage: 1. Copy the macro into a statement tab in the dbt Cloud IDE, or into an analysis file, and compile your code diff --git a/integration_tests/seeds/data__b_relation.csv b/integration_tests/seeds/data__b_relation.csv new file mode 100644 index 0000000..9b9932e --- /dev/null +++ b/integration_tests/seeds/data__b_relation.csv @@ -0,0 +1,3 @@ +col_a,col_b +3,c +4,d \ No newline at end of file diff --git a/integration_tests/tests/test_generate_source.sql b/integration_tests/tests/test_generate_source.sql index a41aed9..875d1f7 100644 --- a/integration_tests/tests/test_generate_source.sql +++ b/integration_tests/tests/test_generate_source.sql @@ -11,6 +11,7 @@ sources: - name: {{ raw_schema | trim }} tables: - name: data__a_relation + - name: data__b_relation {% endset %} diff --git a/integration_tests/tests/test_generate_source_all_args.sql b/integration_tests/tests/test_generate_source_all_args.sql index f0c7967..545ea29 100644 --- a/integration_tests/tests/test_generate_source_all_args.sql +++ b/integration_tests/tests/test_generate_source_all_args.sql @@ -3,6 +3,8 @@ -- test all args {% set actual_source_yaml = codegen.generate_source( schema_name=raw_schema, + table_pattern='%', + exclude='', database_name=target.database, generate_columns=True, include_descriptions=True @@ -10,6 +12,7 @@ {% set expected_source_yaml %} + version: 2 sources: @@ -22,6 +25,13 @@ sources: - name: col_b description: "" + - name: data__b_relation + columns: + - name: col_a + description: "" + - name: col_b + description: "" + {% endset %} {{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} diff --git a/integration_tests/tests/test_generate_source_exclude.sql b/integration_tests/tests/test_generate_source_exclude.sql new file mode 100644 index 0000000..7677628 --- /dev/null +++ b/integration_tests/tests/test_generate_source_exclude.sql @@ -0,0 +1,17 @@ + +{% set raw_schema = generate_schema_name('raw_data') %} + +-- test default args +{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__%', exclude='data__a_%') %} + +{% set expected_source_yaml %} +version: 2 + +sources: + - name: {{ raw_schema | trim }} + tables: + - name: data__b_relation +{% endset %} + + +{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} diff --git a/integration_tests/tests/test_generate_source_table_pattern.sql b/integration_tests/tests/test_generate_source_table_pattern.sql new file mode 100644 index 0000000..823024f --- /dev/null +++ b/integration_tests/tests/test_generate_source_table_pattern.sql @@ -0,0 +1,17 @@ + +{% set raw_schema = generate_schema_name('raw_data') %} + +-- test default args +{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__b_%') %} + +{% set expected_source_yaml %} +version: 2 + +sources: + - name: {{ raw_schema | trim }} + tables: + - name: data__b_relation +{% endset %} + + +{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} diff --git a/macros/generate_source.sql b/macros/generate_source.sql index 9bd6bca..75d4302 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -1,9 +1,10 @@ -{% macro get_tables_in_schema(schema_name, database_name=target.database) %} - +{% macro get_tables_in_schema(schema_name, database_name=target.database, table_pattern='%', exclude='') %} + {% set tables=dbt_utils.get_relations_by_pattern( - database=database_name, - schema_pattern=schema_name, - table_pattern='%' + schema_pattern=schema_name, + database=database_name, + table_pattern=table_pattern, + exclude=exclude ) %} {% set table_list= tables | map(attribute='identifier') %} @@ -14,7 +15,7 @@ --- -{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False) %} +{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='%', exclude='') %} {% set sources_yaml=[] %} @@ -29,7 +30,7 @@ {% do sources_yaml.append(' tables:') %} -{% set tables=codegen.get_tables_in_schema(schema_name, database_name) %} +{% set tables=codegen.get_tables_in_schema(schema_name, database_name, table_pattern, exclude) %} {% for table in tables %} {% do sources_yaml.append(' - name: ' ~ table | lower ) %}