Skip to content

Commit

Permalink
use reference keys instead of relations (#4410)
Browse files Browse the repository at this point in the history
  • Loading branch information
Nathaniel May authored Dec 2, 2021
1 parent c675c2d commit 9bdf5fe
Show file tree
Hide file tree
Showing 7 changed files with 47 additions and 38 deletions.
8 changes: 6 additions & 2 deletions core/dbt/adapters/base/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
ComponentName, BaseRelation, InformationSchema, SchemaSearchMap
)
from dbt.adapters.base import Column as BaseColumn
from dbt.adapters.cache import RelationsCache
from dbt.adapters.cache import RelationsCache, _make_key


SeedModel = Union[ParsedSeedNode, CompiledSeedNode]
Expand Down Expand Up @@ -676,7 +676,11 @@ def list_relations(
relations = self.list_relations_without_caching(
schema_relation
)
fire_event(ListRelations(database=database, schema=schema, relations=relations))
fire_event(ListRelations(
database=database,
schema=schema,
relations=[_make_key(x) for x in relations]
))

return relations

Expand Down
16 changes: 2 additions & 14 deletions core/dbt/adapters/cache.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import threading
from collections import namedtuple
from copy import deepcopy
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple

from dbt.adapters.reference_keys import _make_key, _ReferenceKey
import dbt.exceptions
from dbt.events.functions import fire_event
from dbt.events.types import (
Expand All @@ -22,18 +22,6 @@
)
from dbt.utils import lowercase

_ReferenceKey = namedtuple('_ReferenceKey', 'database schema identifier')


def _make_key(relation) -> _ReferenceKey:
"""Make _ReferenceKeys with lowercase values for the cache so we don't have
to keep track of quoting
"""
# databases and schemas can both be None
return _ReferenceKey(lowercase(relation.database),
lowercase(relation.schema),
lowercase(relation.identifier))


def dot_separated(key: _ReferenceKey) -> str:
"""Return the key in dot-separated string form.
Expand Down Expand Up @@ -334,7 +322,7 @@ def add(self, relation):
:param BaseRelation relation: The underlying relation.
"""
cached = _CachedRelation(relation)
fire_event(AddRelation(relation=cached))
fire_event(AddRelation(relation=_make_key(cached)))
fire_event(DumpBeforeAddGraph(dump=self.dump_graph()))

with self.lock:
Expand Down
24 changes: 24 additions & 0 deletions core/dbt/adapters/reference_keys.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# this module exists to resolve circular imports with the events module

from collections import namedtuple
from typing import Optional


_ReferenceKey = namedtuple('_ReferenceKey', 'database schema identifier')


def lowercase(value: Optional[str]) -> Optional[str]:
if value is None:
return None
else:
return value.lower()


def _make_key(relation) -> _ReferenceKey:
"""Make _ReferenceKeys with lowercase values for the cache so we don't have
to keep track of quoting
"""
# databases and schemas can both be None
return _ReferenceKey(lowercase(relation.database),
lowercase(relation.schema),
lowercase(relation.identifier))
5 changes: 3 additions & 2 deletions core/dbt/adapters/sql/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from dbt.contracts.connection import Connection
import dbt.exceptions
from dbt.adapters.base import BaseAdapter, available
from dbt.adapters.cache import _make_key
from dbt.adapters.sql import SQLConnectionManager
from dbt.events.functions import fire_event
from dbt.events.types import ColTypeChange, SchemaCreation, SchemaDrop
Expand Down Expand Up @@ -182,7 +183,7 @@ def get_columns_in_relation(self, relation):

def create_schema(self, relation: BaseRelation) -> None:
relation = relation.without_identifier()
fire_event(SchemaCreation(relation=relation))
fire_event(SchemaCreation(relation=_make_key(relation)))
kwargs = {
'relation': relation,
}
Expand All @@ -193,7 +194,7 @@ def create_schema(self, relation: BaseRelation) -> None:

def drop_schema(self, relation: BaseRelation) -> None:
relation = relation.without_identifier()
fire_event(SchemaDrop(relation=relation))
fire_event(SchemaDrop(relation=_make_key(relation)))
kwargs = {
'relation': relation,
}
Expand Down
24 changes: 8 additions & 16 deletions core/dbt/events/types.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import argparse
from dataclasses import dataclass
from dbt.adapters.reference_keys import _make_key, _ReferenceKey
from dbt.events.stubs import (
_CachedRelation,
BaseRelation,
ParsedModelNode,
ParsedHookNode,
_ReferenceKey,
RunResult
)
from dbt import ui
Expand Down Expand Up @@ -506,7 +506,7 @@ def message(self) -> str:
class ListRelations(DebugLevel, Cli, File):
database: Optional[str]
schema: str
relations: List[BaseRelation]
relations: List[_ReferenceKey]
code: str = "E014"

def message(self) -> str:
Expand Down Expand Up @@ -573,20 +573,16 @@ def message(self) -> str:

@dataclass
class SchemaCreation(DebugLevel, Cli, File):
relation: BaseRelation
relation: _ReferenceKey
code: str = "E020"

def message(self) -> str:
return f'Creating schema "{self.relation}"'

@classmethod
def asdict(cls, data: list) -> dict:
return dict((k, str(v)) for k, v in data)


@dataclass
class SchemaDrop(DebugLevel, Cli, File):
relation: BaseRelation
relation: _ReferenceKey
code: str = "E021"

def message(self) -> str:
Expand Down Expand Up @@ -625,16 +621,12 @@ def message(self) -> str:

@dataclass
class AddRelation(DebugLevel, Cli, File, Cache):
relation: _CachedRelation
relation: _ReferenceKey
code: str = "E024"

def message(self) -> str:
return f"Adding relation: {str(self.relation)}"

@classmethod
def asdict(cls, data: list) -> dict:
return dict((k, str(v)) for k, v in data)


@dataclass
class DropMissingRelation(DebugLevel, Cli, File, Cache):
Expand Down Expand Up @@ -2499,8 +2491,8 @@ def message(self) -> str:
SQLQueryStatus(status="", elapsed=0.1)
SQLCommit(conn_name="")
ColTypeChange(orig_type="", new_type="", table="")
SchemaCreation(relation=BaseRelation())
SchemaDrop(relation=BaseRelation())
SchemaCreation(relation=_make_key(BaseRelation()))
SchemaDrop(relation=_make_key(BaseRelation()))
UncachedRelation(
dep_key=_ReferenceKey(database="", schema="", identifier=""),
ref_key=_ReferenceKey(database="", schema="", identifier=""),
Expand All @@ -2509,7 +2501,7 @@ def message(self) -> str:
dep_key=_ReferenceKey(database="", schema="", identifier=""),
ref_key=_ReferenceKey(database="", schema="", identifier=""),
)
AddRelation(relation=_CachedRelation())
AddRelation(relation=_make_key(_CachedRelation()))
DropMissingRelation(relation=_ReferenceKey(database="", schema="", identifier=""))
DropCascade(
dropped=_ReferenceKey(database="", schema="", identifier=""),
Expand Down
2 changes: 1 addition & 1 deletion core/dbt/task/deps.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ def run(self):
source_type = package.source_type()
version = package.get_version()

fire_event(DepsStartPackageInstall(package=package))
fire_event(DepsStartPackageInstall(package=package.nice_version_name()))
package.install(self.config, renderer)
fire_event(DepsInstallInfo(version_name=package.nice_version_name()))
if source_type == 'hub':
Expand Down
6 changes: 3 additions & 3 deletions test/unit/test_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,8 @@ def MockNode():
SQLQueryStatus(status="", elapsed=0.1),
SQLCommit(conn_name=""),
ColTypeChange(orig_type="", new_type="", table=""),
SchemaCreation(relation=BaseRelation()),
SchemaDrop(relation=BaseRelation()),
SchemaCreation(relation=_ReferenceKey(database="", schema="", identifier="")),
SchemaDrop(relation=_ReferenceKey(database="", schema="", identifier="")),
UncachedRelation(
dep_key=_ReferenceKey(database="", schema="", identifier=""),
ref_key=_ReferenceKey(database="", schema="", identifier=""),
Expand All @@ -206,7 +206,7 @@ def MockNode():
dep_key=_ReferenceKey(database="", schema="", identifier=""),
ref_key=_ReferenceKey(database="", schema="", identifier=""),
),
AddRelation(relation=_CachedRelation()),
AddRelation(relation=_ReferenceKey(database="", schema="", identifier="")),
DropMissingRelation(relation=_ReferenceKey(database="", schema="", identifier="")),
DropCascade(
dropped=_ReferenceKey(database="", schema="", identifier=""),
Expand Down

0 comments on commit 9bdf5fe

Please sign in to comment.