CT 2196, CT2121 constraints column order (#7161)

.changes/unreleased/Features-20230313-135917.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Make model contracts agnostic to ordering
+time: 2023-03-13T13:59:17.255368-04:00
+custom:
+  Author: gshank
+  Issue: 6975 7064

core/dbt/clients/jinja.py

@@ -483,7 +483,7 @@ def get_environment(
     native: bool = False,
 ) -> jinja2.Environment:
     args: Dict[str, List[Union[str, Type[jinja2.ext.Extension]]]] = {
-        "extensions": ["jinja2.ext.do"]
+        "extensions": ["jinja2.ext.do", "jinja2.ext.loopcontrols"]
     }
 
     if capture_macros:

core/dbt/context/exceptions_jinja.py

@@ -23,6 +23,7 @@
     PropertyYMLError,
     NotImplementedError,
     RelationWrongTypeError,
+    ContractError,
     ColumnTypeMissingError,
 )
 
@@ -66,6 +67,10 @@ def raise_compiler_error(msg, node=None) -> NoReturn:
     raise CompilationError(msg, node)
 
 
+def raise_contract_error(yaml_columns, sql_columns) -> NoReturn:
+    raise ContractError(yaml_columns, sql_columns)
+
+
 def raise_database_error(msg, node=None) -> NoReturn:
     raise DbtDatabaseError(msg, node)
 
@@ -124,6 +129,7 @@ def column_type_missing(column_names) -> NoReturn:
         raise_invalid_property_yml_version,
         raise_not_implemented,
         relation_wrong_type,
+        raise_contract_error,
         column_type_missing,
     ]
 }

core/dbt/exceptions.py

@@ -2124,6 +2124,23 @@ def get_message(self) -> str:
         return msg
 
 
+class ContractError(CompilationError):
+    def __init__(self, yaml_columns, sql_columns):
+        self.yaml_columns = yaml_columns
+        self.sql_columns = sql_columns
+        super().__init__(msg=self.get_message())
+
+    def get_message(self) -> str:
+        msg = (
+            "Contracts are enabled for this model. "
+            "Please ensure the name, data_type, and number of columns in your `yml` file "
+            "match the columns in your SQL file.\n"
+            f"Schema File Columns: {self.yaml_columns}\n"
+            f"SQL File Columns: {self.sql_columns}"
+        )
+        return msg
+
+
 # not modifying these since rpc should be deprecated soon
 class UnknownAsyncIDException(Exception):
     CODE = 10012

core/dbt/include/global_project/macros/materializations/models/table/columns_spec_ddl.sql

@@ -29,31 +29,60 @@
 
 {#
   Compares the column schema provided by a model's sql file to the column schema provided by a model's schema file.
-  If any differences in name, data_type or order of columns exist between the two schemas, raises a compiler error
+  If any differences in name, data_type or number of columns exist between the two schemas, raises a compiler error
 #}
 {% macro assert_columns_equivalent(sql) %}
   {#-- Obtain the column schema provided by sql file. #}
   {%- set sql_file_provided_columns = get_column_schema_from_query(sql) -%}
   {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}
   {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(model['columns'])) -%}
 
-  {%- set sql_file_provided_columns_formatted = format_columns(sql_file_provided_columns)  -%}
-  {%- set schema_file_provided_columns_formatted = format_columns(schema_file_provided_columns)  -%}
+  {#-- create dictionaries with name and formatted data type and strings for exception #}
+  {%- set sql_columns = format_columns(sql_file_provided_columns) -%}
+  {%- set string_sql_columns = stringify_formatted_columns(sql_columns) -%}
+  {%- set yaml_columns = format_columns(schema_file_provided_columns)  -%}
+  {%- set string_yaml_columns = stringify_formatted_columns(yaml_columns) -%}
 
-  {%- if sql_file_provided_columns_formatted != schema_file_provided_columns_formatted -%}
-    {%- do exceptions.raise_compiler_error('Please ensure the name, data_type, order, and number of columns in your `yml` file match the columns in your SQL file.\nSchema File Columns: ' ~ (schema_file_provided_columns_formatted|trim) ~ '\n\nSQL File Columns: ' ~ (sql_file_provided_columns_formatted|trim) ~ ' ' ) %}
+  {%- if sql_columns|length != yaml_columns|length -%}
+    {%- do exceptions.raise_contract_error(string_yaml_columns, string_sql_columns) -%}
   {%- endif -%}
 
+  {%- for sql_col in sql_columns -%}
+    {%- set yaml_col = [] -%}
+    {%- for this_col in yaml_columns -%}
+      {%- if this_col['name'] == sql_col['name'] -%}
+        {%- do yaml_col.append(this_col) -%}
+        {%- break -%}
+      {%- endif -%}
+    {%- endfor -%}
+    {%- if not yaml_col -%}
+      {#-- Column with name not found in yaml #}
+      {%- do exceptions.raise_contract_error(string_yaml_columns, string_sql_columns) -%}
+    {%- endif -%}
+    {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}
+      {#-- Column data types don't match #}
+      {%- do exceptions.raise_contract_error(string_yaml_columns, string_sql_columns) -%}
+    {%- endif -%}
+  {%- endfor -%}
+
 {% endmacro %}
 
 {% macro format_columns(columns) %}
   {% set formatted_columns = [] %}
   {% for column in columns %}
     {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}
-    {%- do formatted_columns.append(formatted_column) -%}
+    {%- do formatted_columns.append({'name': column.name, 'formatted': formatted_column}) -%}
   {% endfor %}
-  {{ return(formatted_columns|join(', ')) }}
-{%- endmacro -%}
+  {{ return(formatted_columns) }}
+{% endmacro %}
+
+{% macro stringify_formatted_columns(formatted_columns) %}
+  {% set column_strings = [] %}
+  {% for column in formatted_columns %}
+     {% do column_strings.append(column['formatted']) %}
+  {% endfor %}
+  {{ return(column_strings|join(', ')) }}
+{% endmacro %}
 
 {% macro default__format_column(column) -%}
   {{ return(column.column.lower() ~ " " ~ column.dtype) }}

core/dbt/include/global_project/macros/materializations/models/table/create_table_as.sql

@@ -25,11 +25,26 @@
 
   create {% if temporary: -%}temporary{%- endif %} table
     {{ relation.include(database=(not temporary), schema=(not temporary)) }}
-    {% if config.get('contract', False) %}
-      {{ get_assert_columns_equivalent(sql) }}
-      {{ get_columns_spec_ddl() }}
-    {% endif %}
+  {% if config.get('contract', False) %}
+    {{ get_assert_columns_equivalent(sql) }}
+    {{ get_columns_spec_ddl() }}
+    {%- set sql = get_select_subquery(sql) %}
+  {% endif %}
   as (
     {{ sql }}
   );
 {%- endmacro %}
+
+{% macro get_select_subquery(sql) %}
+  {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}
+{% endmacro %}
+
+{% macro default__get_select_subquery(sql) %}
+    select
+    {% for column in model['columns'] %}
+      {{ column }}{{ ", " if not loop.last }}
+    {% endfor %}
+    from (
+        {{ sql }}
+    ) as model_subq
+{%- endmacro %}

plugins/postgres/dbt/include/postgres/macros/adapters.sql

@@ -13,10 +13,11 @@
     {{ get_assert_columns_equivalent(sql) }}
     {{ get_columns_spec_ddl() }} ;
     insert into {{ relation }} {{ get_column_names() }}
-    {% else %}
-      as
+    {%- set sql = get_select_subquery(sql) %}
+  {% else %}
+    as
   {% endif %}
-    (
+  (
     {{ sql }}
   );
 {%- endmacro %}

tests/adapter/dbt/tests/adapter/constraints/fixtures.py

@@ -6,9 +6,9 @@
 }}
 
 select
-  1 as id,
   'blue' as color,
-  cast('2019-01-01' as date) as date_day
+  1 as id,
+  '2019-01-01' as date_day
 """
 
 my_model_wrong_order_sql = """
@@ -21,7 +21,7 @@
 select
   'blue' as color,
   1 as id,
-  cast('2019-01-01' as date) as date_day
+  '2019-01-01' as date_day
 """
 
 my_model_wrong_name_sql = """
@@ -32,9 +32,9 @@
 }}
 
 select
-  1 as error,
   'blue' as color,
-  cast('2019-01-01' as date) as date_day
+  1 as error,
+  '2019-01-01' as date_day
 """
 
 my_model_data_type_sql = """
@@ -60,7 +60,7 @@
   cast(null as {{ dbt.type_int() }}) as id,
   -- change the color as well (to test rollback)
   'red' as color,
-  cast('2019-01-01' as date) as date_day
+  '2019-01-01' as date_day
 """
 
 model_schema_yml = """
@@ -81,7 +81,7 @@
       - name: color
         data_type: text
       - name: date_day
-        data_type: date
+        data_type: text
   - name: my_model_error
     config:
       contract: true
@@ -96,7 +96,7 @@
       - name: color
         data_type: text
       - name: date_day
-        data_type: date
+        data_type: text
   - name: my_model_wrong_order
     config:
       contract: true
@@ -111,7 +111,7 @@
       - name: color
         data_type: text
       - name: date_day
-        data_type: date
+        data_type: text
   - name: my_model_wrong_name
     config:
       contract: true
@@ -126,7 +126,7 @@
       - name: color
         data_type: text
       - name: date_day
-        data_type: date
+        data_type: text
 """
 
 model_data_type_schema_yml = """
@@ -150,7 +150,7 @@
 select
   1 as id,
   'blue' as color,
-  cast('2019-01-01' as date) as date_day
+  '2019-01-01' as date_day
 """
 
 my_model_view_wrong_order_sql = """
@@ -163,7 +163,7 @@
 select
   'blue' as color,
   1 as id,
-  cast('2019-01-01' as date) as date_day
+  '2019-01-01' as date_day
 """
 
 my_model_view_wrong_name_sql = """
@@ -174,9 +174,9 @@
 }}
 
 select
-  1 as error,
   'blue' as color,
-  cast('2019-01-01' as date) as date_day
+  1 as error,
+  '2019-01-01' as date_day
 """
 
 my_model_view_with_nulls_sql = """
@@ -191,5 +191,5 @@
   cast(null as {{ dbt.type_int() }}) as id,
   -- change the color as well (to test rollback)
   'red' as color,
-  cast('2019-01-01' as date) as date_day
+  '2019-01-01' as date_day
 """