Skip to content

Commit

Permalink
partial parsing for groups
Browse files Browse the repository at this point in the history
  • Loading branch information
MichelleArk committed Feb 13, 2023
1 parent 24ec6a2 commit 3abb842
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 0 deletions.
1 change: 1 addition & 0 deletions core/dbt/contracts/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ class SchemaSourceFile(BaseSourceFile):
sources: List[str] = field(default_factory=list)
exposures: List[str] = field(default_factory=list)
metrics: List[str] = field(default_factory=list)
groups: List[str] = field(default_factory=list)
# node patches contain models, seeds, snapshots, analyses
ndp: List[str] = field(default_factory=list)
# any macro patches in this file by macro unique_id.
Expand Down
2 changes: 2 additions & 0 deletions core/dbt/contracts/graph/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -1074,6 +1074,8 @@ def add_node(self, source_file: AnySourceFile, node: ManifestNode, test_from=Non
source_file.metrics.append(node.unique_id)
if isinstance(node, Exposure):
source_file.exposures.append(node.unique_id)
if isinstance(node, Group):
source_file.groups.append(node.unique_id)
else:
source_file.nodes.append(node.unique_id)

Expand Down
36 changes: 36 additions & 0 deletions core/dbt/parser/partial.py
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,29 @@ def handle_schema_file_changes(self, schema_file, saved_yaml_dict, new_yaml_dict
self.delete_schema_metric(schema_file, elem)
self.merge_patch(schema_file, dict_key, elem)

# groups
dict_key = "groups"
group_diff = self.get_diff_for("groups", saved_yaml_dict, new_yaml_dict)
if group_diff["changed"]:
for group in group_diff["changed"]:
self.delete_schema_group(schema_file, group)
self.merge_patch(schema_file, dict_key, group)
if group_diff["deleted"]:
for metric in group_diff["deleted"]:
self.delete_schema_group(schema_file, group)
if group_diff["added"]:
for metric in group_diff["added"]:
self.merge_patch(schema_file, dict_key, group)
# Handle schema file updates due to env_var changes
if dict_key in env_var_changes and dict_key in new_yaml_dict:
for name in env_var_changes[dict_key]:
if name in group_diff["changed_or_deleted_names"]:
continue
elem = self.get_schema_element(new_yaml_dict[dict_key], name)
if elem:
self.delete_schema_group(schema_file, elem)
self.merge_patch(schema_file, dict_key, elem)

# Take a "section" of the schema file yaml dictionary from saved and new schema files
# and determine which parts have changed
def get_diff_for(self, key, saved_yaml_dict, new_yaml_dict):
Expand Down Expand Up @@ -903,6 +926,19 @@ def delete_schema_exposure(self, schema_file, exposure_dict):
elif unique_id in self.saved_manifest.disabled:
self.delete_disabled(unique_id, schema_file.file_id)

# groups are created only from schema files, so just delete the group
def delete_schema_group(self, schema_file, group_dict):
group_name = group_dict["name"]
groups = schema_file.groups.copy()
for unique_id in groups:
if unique_id in self.saved_manifest.groups:
group = self.saved_manifest.groups[unique_id]
if group.name == group_name:
self.deleted_manifest.groups[unique_id] = self.saved_manifest.groups.pop(
unique_id
)
schema_file.groups.remove(unique_id)

# metrics are created only from schema files, but also can be referred to by other nodes
def delete_schema_metric(self, schema_file, metric_dict):
metric_name = metric_dict["name"]
Expand Down

0 comments on commit 3abb842

Please sign in to comment.