From 3abb842b32b5c8312ee20f282edec0cb9c1c0e48 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 9 Feb 2023 11:48:44 -0500 Subject: [PATCH] partial parsing for groups --- core/dbt/contracts/files.py | 1 + core/dbt/contracts/graph/manifest.py | 2 ++ core/dbt/parser/partial.py | 36 ++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index 93f12a1411e..9e82247da00 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -227,6 +227,7 @@ class SchemaSourceFile(BaseSourceFile): sources: List[str] = field(default_factory=list) exposures: List[str] = field(default_factory=list) metrics: List[str] = field(default_factory=list) + groups: List[str] = field(default_factory=list) # node patches contain models, seeds, snapshots, analyses ndp: List[str] = field(default_factory=list) # any macro patches in this file by macro unique_id. diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 95cfaa79d81..9db52962ecf 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -1074,6 +1074,8 @@ def add_node(self, source_file: AnySourceFile, node: ManifestNode, test_from=Non source_file.metrics.append(node.unique_id) if isinstance(node, Exposure): source_file.exposures.append(node.unique_id) + if isinstance(node, Group): + source_file.groups.append(node.unique_id) else: source_file.nodes.append(node.unique_id) diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index d6afe223278..03caefb1907 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -746,6 +746,29 @@ def handle_schema_file_changes(self, schema_file, saved_yaml_dict, new_yaml_dict self.delete_schema_metric(schema_file, elem) self.merge_patch(schema_file, dict_key, elem) + # groups + dict_key = "groups" + group_diff = self.get_diff_for("groups", saved_yaml_dict, new_yaml_dict) + if group_diff["changed"]: + for group in group_diff["changed"]: + self.delete_schema_group(schema_file, group) + self.merge_patch(schema_file, dict_key, group) + if group_diff["deleted"]: + for metric in group_diff["deleted"]: + self.delete_schema_group(schema_file, group) + if group_diff["added"]: + for metric in group_diff["added"]: + self.merge_patch(schema_file, dict_key, group) + # Handle schema file updates due to env_var changes + if dict_key in env_var_changes and dict_key in new_yaml_dict: + for name in env_var_changes[dict_key]: + if name in group_diff["changed_or_deleted_names"]: + continue + elem = self.get_schema_element(new_yaml_dict[dict_key], name) + if elem: + self.delete_schema_group(schema_file, elem) + self.merge_patch(schema_file, dict_key, elem) + # Take a "section" of the schema file yaml dictionary from saved and new schema files # and determine which parts have changed def get_diff_for(self, key, saved_yaml_dict, new_yaml_dict): @@ -903,6 +926,19 @@ def delete_schema_exposure(self, schema_file, exposure_dict): elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) + # groups are created only from schema files, so just delete the group + def delete_schema_group(self, schema_file, group_dict): + group_name = group_dict["name"] + groups = schema_file.groups.copy() + for unique_id in groups: + if unique_id in self.saved_manifest.groups: + group = self.saved_manifest.groups[unique_id] + if group.name == group_name: + self.deleted_manifest.groups[unique_id] = self.saved_manifest.groups.pop( + unique_id + ) + schema_file.groups.remove(unique_id) + # metrics are created only from schema files, but also can be referred to by other nodes def delete_schema_metric(self, schema_file, metric_dict): metric_name = metric_dict["name"]