fix: Use internal macro pilo_star as a workaround for star()

While waiting for dbt_utils.star() to be fixed with dbt-labs/dbt-utils#732, we will use this proxy macro to not having to repeat the tedious if/else in all of our scripts.
gip-inclusion · May 31, 2023 · 022beab · 022beab
1 parent 972f620
commit 022beab
Show file tree

Hide file tree

Showing 12 changed files with 29 additions and 67 deletions.
diff --git a/dbt/macros/pilo_star.sql b/dbt/macros/pilo_star.sql
@@ -0,0 +1,15 @@
+{% macro pilo_star(table_name, relation_alias=False, except=[], prefix='', suffix='', quote_identifiers=True) -%}
+    /* FIXME(vperron): Normally, sqlfluff rendering star() should render a `*`.
+    But for some reason it does not, yet, or not anymore.
+    https://github.com/dbt-labs/dbt-utils/pull/732
+    In the meantime, and since we don't want to fully maintain an intialized DB in CI,
+    we have to workaround it. I would have like to monkeypatch it in CI (some override of
+    the macro somewhere) but could not find a way to do it. Essentialy, we would want the
+    following code being applied automatically. */
+    {%- if env_var('CI', '') -%}
+        id
+    {%- else -%}
+        {{ dbt_utils.star(table_name, relation_alias, except, prefix, suffix, quote_identifiers) }}
+    {%- endif -%}
+{% endmacro %}
+
diff --git a/dbt/models/marts/candidats_auto_prescription.sql b/dbt/models/marts/candidats_auto_prescription.sql
@@ -1,9 +1,5 @@
 select
-    {% if env_var('CI', '') %}
-        id,
-    {% else %}
-        {{ dbt_utils.star(ref('stg_candidats_autoprescription'), relation_alias="autopr_c") }},
-    {% endif %}
+    {{ pilo_star(ref('stg_candidats_autoprescription'), relation_alias="autopr_c") }},
     ac.total_candidats,
     s.nom_structure_complet as "nom_structure_complet"
 from

diff --git a/dbt/models/marts/candidatures_dihal.sql b/dbt/models/marts/candidatures_dihal.sql
@@ -1,8 +1,4 @@
 select
-    {% if env_var('CI', '') %}
-        id
-    {% else %}
-        {{ dbt_utils.star(ref('candidatures_echelle_locale')) }}
-    {% endif %}
+    {{ pilo_star(ref('candidatures_echelle_locale')) }}
 from {{ ref('candidatures_echelle_locale') }}
 where "type_org_prescripteur" in ('CHRS', 'CHU', 'RS_FJT', 'OIL')
diff --git a/dbt/models/marts/candidatures_echelle_locale.sql b/dbt/models/marts/candidatures_echelle_locale.sql
@@ -1,13 +1,8 @@
 select
-    {% if env_var('CI', '') %}
-        id,
-    {% else %}
-        {{ dbt_utils.star(ref('stg_candidatures'), relation_alias='candidatures') }},
-        {{ dbt_utils.star(ref('stg_org_prescripteur'), except=["id_org"], relation_alias='org_prescripteur') }},
-        {{ dbt_utils.star(ref('stg_bassin_emploi'),
-        except=["nom_departement", "nom_region", "type_epci", "id_structure"], relation_alias='bassin_emploi') }},
-        {{ dbt_utils.star(ref('stg_reseaux'), except=["SIRET","id_structure"], relation_alias='rsx') }},
-    {% endif %}
+    {{ pilo_star(ref('stg_candidatures'), relation_alias='candidatures') }},
+    {{ pilo_star(ref('stg_org_prescripteur'), except=["id_org"], relation_alias='org_prescripteur') }},
+    {{ pilo_star(ref('stg_bassin_emploi'), except=["nom_departement", "nom_region", "type_epci", "id_structure"], relation_alias='bassin_emploi') }},
+    {{ pilo_star(ref('stg_reseaux'), except=["SIRET","id_structure"], relation_alias='rsx') }},
     case
         when candidatures.injection_ai = 0 then 'Non'
         else 'Oui'

diff --git a/dbt/models/marts/organisations_dihal.sql b/dbt/models/marts/organisations_dihal.sql
@@ -1,8 +1,4 @@
 select
-    {% if env_var('CI', '') %}
-        id
-    {% else %}
-        {{ dbt_utils.star(ref('stg_organisations')) }}
-    {% endif %}
+    {{ pilo_star(ref('stg_organisations')) }}
 from {{ ref('stg_organisations') }}
 where type in ('CHRS', 'CHU', 'RS_FJT', 'OIL')
diff --git a/dbt/models/marts/reseaux.sql b/dbt/models/marts/reseaux.sql
@@ -1,8 +1,4 @@
 select
-    {% if env_var('CI', '') %}
-        id
-    {% else %}
-        {{ dbt_utils.star(ref('stg_reseaux')) }}
-    {% endif %}
+    {{ pilo_star(ref('stg_reseaux')) }}
 from
     {{ ref('stg_reseaux') }}
diff --git a/dbt/models/marts/suivi_auto_prescription.sql b/dbt/models/marts/suivi_auto_prescription.sql
@@ -1,9 +1,5 @@
 select
-    {% if env_var('CI', '') %}
-        autopr_all.*,
-    {% else %}
-        {{ dbt_utils.star(ref('stg_candidatures_autoprescription')) }},
-    {% endif %}
+    {{ pilo_star(ref('stg_candidatures_autoprescription')) }},
     s.siret                 as siret,
     s.active                as active,
     s.ville                 as ville,

diff --git a/dbt/models/marts/suivi_candidatures_prescripteurs_habilites.sql b/dbt/models/marts/suivi_candidatures_prescripteurs_habilites.sql
@@ -1,17 +1,6 @@
 with candidatures_ph as (
     select
-        /* FIXME(vperron): Normally, sqlfluff rendering star() should render a `*`.
-        But for some reason it does not, yet, or not anymore.
-        https://github.com/dbt-labs/dbt-utils/pull/732
-        In the meantime, and since we don't want to fully maintain an intialized DB in CI,
-        we have to workaround it. I would have like to monkeypatch it in CI (some override of
-        the macro somewhere) but could not find a way to do it. Essentialy, we would want the
-        following code being applied automatically. */
-        {% if env_var('CI', '') %}
-            *,
-        {% else %}
-            {{ dbt_utils.star(ref('candidatures_echelle_locale')) }},
-        {% endif %}
+        {{ pilo_star(ref('candidatures_echelle_locale')) }},
         replace("origine_détaillée", 'Prescripteur habilité ', '') as origine_simplifiee
     from {{ ref('candidatures_echelle_locale') }}
     where starts_with("origine_détaillée", 'Prescripteur habilité')

diff --git a/dbt/models/staging/stg_candidats_autoprescription.sql b/dbt/models/staging/stg_candidats_autoprescription.sql
@@ -1,9 +1,5 @@
 select distinct
-    {% if env_var('CI', '') %}
-        id,
-    {% else %}
-        {{ dbt_utils.star(source('emplois', 'candidats'), relation_alias="c") }},
-    {% endif %}
+    {{ pilo_star(source('emplois', 'candidats'), relation_alias="c") }},
     cd."état",
     cd.nom_structure,
     cd.type_structure,

diff --git a/dbt/models/staging/stg_candidatures.sql b/dbt/models/staging/stg_candidatures.sql
@@ -1,10 +1,5 @@
 select
-    {% if env_var('CI', '') %}
-        id,
-    {% else %}
-        {{ dbt_utils.star(source('emplois', 'candidatures'),
-            except=["état", "motif_de_refus", "origine", "délai_de_réponse", "délai_prise_en_compte"]) }},
-    {% endif %}
+    {{ pilo_star(source('emplois', 'candidatures'), except=["état", "motif_de_refus", "origine", "délai_de_réponse", "délai_prise_en_compte"]) }},
     case
         when "état" = 'Candidature déclinée' then 'Candidature refusée'
         else "état"

diff --git a/dbt/models/staging/stg_organisations.sql b/dbt/models/staging/stg_organisations.sql
@@ -1,9 +1,5 @@
 select
-    {% if env_var('CI', '') %}
-        id,
-    {% else %}
-        {{ dbt_utils.star(source('emplois', 'organisations')) }},
-    {% endif %}
+    {{ pilo_star(source('emplois', 'organisations')) }},
     case
         when "habilitée" = 1 then concat('Prescripteur habilité ', "type")
         when "habilitée" = 0 then concat('Orienteur ', "type")

diff --git a/dbt/models/staging/stg_reseaux.sql b/dbt/models/staging/stg_reseaux.sql
@@ -1,9 +1,5 @@
 select
-    {% if env_var('CI', '') %}
-        siret,
-    {% else %}
-        {{ dbt_utils.star(source('oneshot', 'reseau_iae_adherents')) }},
-    {% endif %}
+    {{ pilo_star(source('oneshot', 'reseau_iae_adherents')) }},
     s.id as id_structure,
     rid.id_institution
 from {{ source('oneshot', 'reseau_iae_adherents') }} as ria