ADAP-814: Add support for replacing materialized views with tables/vi…

…ews and vice versa (#8449)

* first draft of adding in table - materialized view swap
* table/view/materialized view can all replace each other
* update renameable relations to a config
* migrate relations macros from `macros/adapters/relations` to `macros/relations` so that generics are close to the relation specific macros that they reference; also aligns with adapter macro files structure, to look more familiar
* move drop macro to drop macro file
* align the behavior of get_drop_sql and drop_relation, adopt existing default from drop_relation
* add explicit ddl for drop statements instead of inheriting the default from dbt-core
* update replace macro dependent macros to align with naming standards
* update type for mashumaro, update related test

.changes/unreleased/Fixes-20230817-185739.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Add support for swapping materialized views with tables/views and vice versa
+time: 2023-08-17T18:57:39.01958-04:00
+custom:
+  Author: mikealfare
+  Issue: "8449"

core/dbt/adapters/base/relation.py

@@ -1,6 +1,6 @@
 from collections.abc import Hashable
 from dataclasses import dataclass, field
-from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set
+from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set, List
 
 from dbt.contracts.graph.nodes import SourceDefinition, ManifestNode, ResultNode, ParsedNode
 from dbt.contracts.relation import (
@@ -35,6 +35,10 @@ class BaseRelation(FakeAPIObject, Hashable):
     include_policy: Policy = field(default_factory=lambda: Policy())
     quote_policy: Policy = field(default_factory=lambda: Policy())
     dbt_created: bool = False
+    # register relation types that can be renamed for the purpose of replacing relations using stages and backups
+    renameable_relations: List[str] = field(
+        default_factory=lambda: [RelationType.Table, RelationType.View]
+    )
 
     def _is_exactish_match(self, field: ComponentName, value: str) -> bool:
         if self.dbt_created and self.quote_policy.get_part(field) is False:
@@ -169,7 +173,6 @@ def without_identifier(self) -> "BaseRelation":
         return self.include(identifier=False).replace_path(identifier=None)
 
     def _render_iterator(self) -> Iterator[Tuple[Optional[ComponentName], Optional[str]]]:
-
         for key in ComponentName:
             path_part: Optional[str] = None
             if self.include_policy.get_part(key):
@@ -286,6 +289,10 @@ def create(
         )
         return cls.from_dict(kwargs)
 
+    @property
+    def can_be_renamed(self):
+        return self.type in self.renameable_relations
+
     def __repr__(self) -> str:
         return "<{} {}>".format(self.__class__.__name__, self.render())
 

core/dbt/include/global_project/macros/adapters/relation.sql

@@ -77,10 +77,3 @@
 {% macro load_relation(relation) %}
     {{ return(load_cached_relation(relation)) }}
 {% endmacro %}
-
-
-{% macro drop_relation_if_exists(relation) %}
-  {% if relation is not none %}
-    {{ adapter.drop_relation(relation) }}
-  {% endif %}
-{% endmacro %}

core/dbt/include/global_project/macros/materializations/models/materialized_view/materialized_view.sql

@@ -57,7 +57,7 @@
     {% if existing_relation is none %}
         {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}
     {% elif full_refresh_mode or not existing_relation.is_materialized_view %}
-        {% set build_sql = get_replace_materialized_view_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) %}
+        {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}
     {% else %}
 
         -- get config options

core/dbt/include/global_project/macros/materializations/models/view/create_or_replace_view.sql

@@ -1,7 +1,7 @@
 /* {#
        Core materialization implementation. BigQuery and Snowflake are similar
-       because both can use `create or replace view` where the resulting view schema
-       is not necessarily the same as the existing view. On Redshift, this would
+       because both can use `create or replace view` where the resulting view's columns
+       are not necessarily the same as those of the existing view. On Redshift, this would
        result in: ERROR:  cannot change number of columns in view
 
        This implementation is superior to the create_temp, swap_with_existing, drop_old

core/dbt/include/global_project/macros/relations/create.sql

@@ -0,0 +1,17 @@
+{%- macro get_create_sql(relation, sql) -%}
+    {{- log('Applying CREATE to: ' ~ relation) -}}
+    {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}
+{%- endmacro -%}
+
+
+{%- macro default__get_create_sql(relation, sql) -%}
+
+    {%- if relation.is_materialized_view -%}
+        {{ get_create_materialized_view_as_sql(relation, sql) }}
+
+    {%- else -%}
+        {{- exceptions.raise_compiler_error("`get_create_sql` has not been implemented for: " ~ relation.type ) -}}
+
+    {%- endif -%}
+
+{%- endmacro -%}

core/dbt/include/global_project/macros/relations/create_backup.sql

@@ -0,0 +1,17 @@
+{%- macro get_create_backup_sql(relation) -%}
+    {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}
+    {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}
+{%- endmacro -%}
+
+
+{%- macro default__get_create_backup_sql(relation) -%}
+
+    -- get the standard backup name
+    {% set backup_relation = make_backup_relation(relation, relation.type) %}
+
+    -- drop any pre-existing backup
+    {{ get_drop_sql(backup_relation) }};
+
+    {{ get_rename_sql(relation, backup_relation.identifier) }}
+
+{%- endmacro -%}

core/dbt/include/global_project/macros/relations/create_intermediate.sql

@@ -0,0 +1,17 @@
+{%- macro get_create_intermediate_sql(relation, sql) -%}
+    {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}
+    {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}
+{%- endmacro -%}
+
+
+{%- macro default__get_create_intermediate_sql(relation, sql) -%}
+
+    -- get the standard intermediate name
+    {% set intermediate_relation = make_intermediate_relation(relation) %}
+
+    -- drop any pre-existing intermediate
+    {{ get_drop_sql(intermediate_relation) }};
+
+    {{ get_create_sql(intermediate_relation, sql) }}
+
+{%- endmacro -%}

core/dbt/include/global_project/macros/relations/drop.sql

@@ -1,17 +1,41 @@
+{%- macro get_drop_sql(relation) -%}
+    {{- log('Applying DROP to: ' ~ relation) -}}
+    {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}
+{%- endmacro -%}
+
+
+{%- macro default__get_drop_sql(relation) -%}
+
+    {%- if relation.is_view -%}
+        {{ drop_view(relation) }}
+
+    {%- elif relation.is_table -%}
+        {{ drop_table(relation) }}
+
+    {%- elif relation.is_materialized_view -%}
+        {{ drop_materialized_view(relation) }}
+
+    {%- else -%}
+        drop {{ relation.type }} if exists {{ relation }} cascade
+
+    {%- endif -%}
+
+{%- endmacro -%}
+
+
 {% macro drop_relation(relation) -%}
     {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}
 {% endmacro %}
 
 {% macro default__drop_relation(relation) -%}
     {% call statement('drop_relation', auto_begin=False) -%}
-        {%- if relation.is_table -%}
-            {{- drop_table(relation) -}}
-        {%- elif relation.is_view -%}
-            {{- drop_view(relation) -}}
-        {%- elif relation.is_materialized_view -%}
-            {{- drop_materialized_view(relation) -}}
-        {%- else -%}
-            drop {{ relation.type }} if exists {{ relation }} cascade
-        {%- endif -%}
+        {{ get_drop_sql(relation) }}
     {%- endcall %}
 {% endmacro %}
+
+
+{% macro drop_relation_if_exists(relation) %}
+  {% if relation is not none %}
+    {{ adapter.drop_relation(relation) }}
+  {% endif %}
+{% endmacro %}

core/dbt/include/global_project/macros/relations/drop_backup.sql

@@ -0,0 +1,14 @@
+{%- macro get_drop_backup_sql(relation) -%}
+    {{- log('Applying DROP BACKUP to: ' ~ relation) -}}
+    {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}
+{%- endmacro -%}
+
+
+{%- macro default__get_drop_backup_sql(relation) -%}
+
+    -- get the standard backup name
+    {% set backup_relation = make_backup_relation(relation, relation.type) %}
+
+    {{ get_drop_sql(backup_relation) }}
+
+{%- endmacro -%}

core/dbt/include/global_project/macros/adapters/relations/materialized_view/create.sql → core/dbt/include/global_project/macros/relations/materialized_view/create.sql

@@ -1,5 +1,4 @@
 {% macro get_create_materialized_view_as_sql(relation, sql) -%}
-    {{- log('Applying CREATE to: ' ~ relation) -}}
     {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}
 {%- endmacro %}
 

core/dbt/include/global_project/macros/adapters/relations/materialized_view/drop.sql → core/dbt/include/global_project/macros/relations/materialized_view/drop.sql

@@ -5,9 +5,10 @@ actually executes the drop, and `get_drop_sql`, which returns the template.
 */ #}
 
 {% macro drop_materialized_view(relation) -%}
-  {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }}
+    {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }}
 {%- endmacro %}
 
+
 {% macro default__drop_materialized_view(relation) -%}
     drop materialized view if exists {{ relation }} cascade
 {%- endmacro %}

core/dbt/include/global_project/macros/relations/materialized_view/rename.sql

@@ -0,0 +1,10 @@
+{% macro get_rename_materialized_view_sql(relation, new_name) %}
+    {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}
+{% endmacro %}
+
+
+{% macro default__get_rename_materialized_view_sql(relation, new_name) %}
+    {{ exceptions.raise_compiler_error(
+        "`get_rename_materialized_view_sql` has not been implemented for this adapter."
+    ) }}
+{% endmacro %}

core/dbt/include/global_project/macros/relations/rename.sql

@@ -1,3 +1,28 @@
+{%- macro get_rename_sql(relation, new_name) -%}
+    {{- log('Applying RENAME to: ' ~ relation) -}}
+    {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}
+{%- endmacro -%}
+
+
+{%- macro default__get_rename_sql(relation, new_name) -%}
+
+    {%- if relation.is_view -%}
+        {{ get_rename_view_sql(relation, new_name) }}
+
+    {%- elif relation.is_table -%}
+        {{ get_rename_table_sql(relation, new_name) }}
+
+    {%- elif relation.is_materialized_view -%}
+        {{ get_rename_materialized_view_sql(relation, new_name) }}
+
+    {%- else -%}
+        {{- exceptions.raise_compiler_error("`get_rename_sql` has not been implemented for: " ~ relation.type ) -}}
+
+    {%- endif -%}
+
+{%- endmacro -%}
+
+
 {% macro rename_relation(from_relation, to_relation) -%}
   {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}
 {% endmacro %}

core/dbt/include/global_project/macros/relations/rename_intermediate.sql

@@ -0,0 +1,14 @@
+{%- macro get_rename_intermediate_sql(relation) -%}
+    {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}
+    {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}
+{%- endmacro -%}
+
+
+{%- macro default__get_rename_intermediate_sql(relation) -%}
+
+    -- get the standard intermediate name
+    {% set intermediate_relation = make_intermediate_relation(relation) %}
+
+    {{ get_rename_sql(intermediate_relation, relation.identifier) }}
+
+{%- endmacro -%}

core/dbt/include/global_project/macros/relations/replace.sql

@@ -0,0 +1,35 @@
+{% macro get_replace_sql(existing_relation, target_relation, sql) %}
+    {{- log('Applying REPLACE to: ' ~ existing_relation) -}}
+    {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}
+{% endmacro %}
+
+
+{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}
+
+    {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}
+    {%- if target_relation.can_be_renamed and existing_relation.can_be_renamed -%}
+        {{ get_create_intermediate_sql(target_relation, sql) }};
+        {{ get_create_backup_sql(existing_relation) }};
+        {{ get_rename_intermediate_sql(target_relation) }};
+        {{ get_drop_backup_sql(existing_relation) }}
+
+    {# /* create target_relation as an intermediate relation, then swap it out with the existing one using drop */ #}
+    {%- elif target_relation.can_be_renamed -%}
+        {{ get_create_intermediate_sql(target_relation, sql) }};
+        {{ get_drop_sql(existing_relation) }};
+        {{ get_rename_intermediate_sql(target_relation) }}
+
+    {# /* create target_relation in place by first backing up the existing relation */ #}
+    {%- elif existing_relation.can_be_renamed -%}
+        {{ get_create_backup_sql(existing_relation) }};
+        {{ get_create_sql(target_relation, sql) }};
+        {{ get_drop_backup_sql(existing_relation) }}
+
+    {# /* no renaming is allowed, so just drop and create */ #}
+    {%- else -%}
+        {{ get_drop_sql(existing_relation) }};
+        {{ get_create_sql(target_relation, sql) }}
+
+    {%- endif -%}
+
+{% endmacro %}

core/dbt/include/global_project/macros/adapters/relations/table/drop.sql → core/dbt/include/global_project/macros/relations/table/drop.sql

@@ -5,9 +5,10 @@ actually executes the drop, and `get_drop_sql`, which returns the template.
 */ #}
 
 {% macro drop_table(relation) -%}
-  {{ return(adapter.dispatch('drop_table', 'dbt')(relation)) }}
+    {{ return(adapter.dispatch('drop_table', 'dbt')(relation)) }}
 {%- endmacro %}
 
+
 {% macro default__drop_table(relation) -%}
     drop table if exists {{ relation }} cascade
 {%- endmacro %}

core/dbt/include/global_project/macros/relations/table/rename.sql

@@ -0,0 +1,10 @@
+{% macro get_rename_table_sql(relation, new_name) %}
+    {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}
+{% endmacro %}
+
+
+{% macro default__get_rename_table_sql(relation, new_name) %}
+    {{ exceptions.raise_compiler_error(
+        "`get_rename_table_sql` has not been implemented for this adapter."
+    ) }}
+{% endmacro %}

core/dbt/include/global_project/macros/adapters/relations/view/drop.sql → core/dbt/include/global_project/macros/relations/view/drop.sql

@@ -5,9 +5,10 @@ actually executes the drop, and `get_drop_sql`, which returns the template.
 */ #}
 
 {% macro drop_view(relation) -%}
-  {{ return(adapter.dispatch('drop_view', 'dbt')(relation)) }}
+    {{ return(adapter.dispatch('drop_view', 'dbt')(relation)) }}
 {%- endmacro %}
 
+
 {% macro default__drop_view(relation) -%}
     drop view if exists {{ relation }} cascade
 {%- endmacro %}

core/dbt/include/global_project/macros/relations/view/rename.sql

@@ -0,0 +1,10 @@
+{% macro get_rename_view_sql(relation, new_name) %}
+    {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}
+{% endmacro %}
+
+
+{% macro default__get_rename_view_sql(relation, new_name) %}
+    {{ exceptions.raise_compiler_error(
+        "`get_rename_view_sql` has not been implemented for this adapter."
+    ) }}
+{% endmacro %}

plugins/postgres/dbt/adapters/postgres/relation.py

@@ -7,6 +7,7 @@
     RelationResults,
 )
 from dbt.context.providers import RuntimeConfigObject
+from dbt.contracts.relation import RelationType
 from dbt.exceptions import DbtRuntimeError
 
 from dbt.adapters.postgres.relation_configs import (
@@ -20,6 +21,12 @@
 
 @dataclass(frozen=True, eq=False, repr=False)
 class PostgresRelation(BaseRelation):
+    relations_that_can_be_renamed = [
+        RelationType.View,
+        RelationType.Table,
+        RelationType.MaterializedView,
+    ]
+
     def __post_init__(self):
         # Check for length of Postgres table/view names.
         # Check self.type to exclude test relation identifiers

plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/alter.sql

@@ -10,7 +10,7 @@
     -- apply a full refresh immediately if needed
     {% if configuration_changes.requires_full_refresh %}
 
-        {{ get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) }}
+        {{ get_replace_sql(existing_relation, relation, sql) }}
 
     -- otherwise apply individual changes as needed
     {% else %}

plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/drop.sql

@@ -0,0 +1,3 @@
+{% macro postgres__drop_materialized_view(relation) -%}
+    drop materialized view if exists {{ relation }} cascade
+{%- endmacro %}

plugins/postgres/dbt/include/postgres/macros/relations/table/drop.sql

@@ -0,0 +1,3 @@
+{% macro postgres__drop_table(relation) -%}
+    drop table if exists {{ relation }} cascade
+{%- endmacro %}

plugins/postgres/dbt/include/postgres/macros/relations/table/rename.sql

@@ -0,0 +1,3 @@
+{% macro postgres__get_rename_table_sql(relation, new_name) %}
+    alter table {{ relation }} rename to {{ new_name }}
+{% endmacro %}

plugins/postgres/dbt/include/postgres/macros/relations/view/drop.sql

@@ -0,0 +1,3 @@
+{% macro postgres__drop_view(relation) -%}
+    drop view if exists {{ relation }} cascade
+{%- endmacro %}

plugins/postgres/dbt/include/postgres/macros/relations/view/rename.sql

@@ -0,0 +1,3 @@
+{% macro postgres__get_rename_view_sql(relation, new_name) %}
+    alter view {{ relation }} rename to {{ new_name }}
+{% endmacro %}