Add utils module

.changes/unreleased/Features-20231026-110821.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: migrate utils to common and adapters folders
+time: 2023-10-26T11:08:21.458709-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "8924"

.flake8

@@ -10,3 +10,5 @@ ignore =
     E741
     E501 # long line checking is done in black
 exclude = test/
+per-file-ignores =
+    */__init__.py: F401

core/dbt/adapters/base/connections.py

@@ -50,7 +50,7 @@
 )
 from dbt.common.events.contextvars import get_node_info
 from dbt import flags
-from dbt.utils import cast_to_str
+from dbt.common.utils import cast_to_str
 
 SleepTime = Union[int, float]  # As taken by time.sleep.
 AdapterHandle = Any  # Adapter connection handle objects can be any class.

core/dbt/adapters/base/impl.py

@@ -65,7 +65,7 @@
     ConstraintNotSupported,
     ConstraintNotEnforced,
 )
-from dbt.utils import filter_null_values, executor, cast_to_str, AttrDict
+from dbt.common.utils import filter_null_values, executor, cast_to_str, AttrDict
 
 from dbt.adapters.base.connections import Connection, AdapterResponse, BaseConnectionManager
 from dbt.adapters.base.meta import AdapterMeta, available

core/dbt/adapters/base/relation.py

@@ -17,7 +17,8 @@
     MultipleDatabasesNotAllowedError,
 )
 from dbt.node_types import NodeType
-from dbt.utils import filter_null_values, deep_merge, classproperty
+from dbt.common.utils import filter_null_values, deep_merge
+from dbt.adapters.utils import classproperty
 
 import dbt.exceptions
 
@@ -246,7 +247,7 @@ def create_from_node(
         if quote_policy is None:
             quote_policy = {}
 
-        quote_policy = dbt.utils.merge(config.quoting, quote_policy)
+        quote_policy = dbt.common.utils.merge(config.quoting, quote_policy)
 
         return cls.create(
             database=node.database,

core/dbt/adapters/contracts/connection.py

@@ -24,7 +24,7 @@
     ValidatedStringMixin,
 )
 from dbt.common.contracts.util import Replaceable
-from dbt.common.util import md5
+from dbt.common.utils import md5
 
 # TODO: dbt.common.events dependency
 from dbt.common.events.functions import fire_event

core/dbt/adapters/relation_configs/config_base.py

@@ -2,7 +2,7 @@
 from typing import Union, Dict
 
 import agate
-from dbt.utils import filter_null_values
+from dbt.common.utils import filter_null_values
 
 
 """

core/dbt/adapters/sql/connections.py

@@ -11,7 +11,7 @@
 from dbt.common.events.functions import fire_event
 from dbt.common.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
 from dbt.common.events.contextvars import get_node_info
-from dbt.utils import cast_to_str
+from dbt.common.utils import cast_to_str
 
 
 class SQLConnectionManager(BaseConnectionManager):

core/dbt/adapters/utils.py

@@ -53,3 +53,16 @@ def translate_aliases(
     """
     translator = Translator(aliases, recurse)
     return translator.translate(kwargs)
+
+
+# some types need to make constants available to the jinja context as
+# attributes, and regular properties only work with objects. maybe this should
+# be handled by the RelationProxy?
+
+
+class classproperty(object):
+    def __init__(self, func) -> None:
+        self.func = func
+
+    def __get__(self, obj, objtype):
+        return self.func(objtype)

core/dbt/cli/requires.py

@@ -30,7 +30,7 @@
 from dbt.parser.manifest import ManifestLoader, write_manifest
 from dbt.profiler import profiler
 from dbt.tracking import active_user, initialize_from_flags, track_run
-from dbt.utils import cast_dict_to_dict_of_strings
+from dbt.common.utils import cast_dict_to_dict_of_strings
 from dbt.plugins import set_up_plugin_manager, get_plugin_manager
 
 from click import Context

core/dbt/clients/jinja.py

@@ -21,9 +21,8 @@
     get_docs_macro_name,
     get_materialization_macro_name,
     get_test_macro_name,
-    deep_map_render,
 )
-
+from dbt.common.utils import deep_map_render
 from dbt.clients._jinja_blocks import BlockIterator, BlockData, BlockTag
 from dbt.contracts.graph.nodes import GenericTestNode
 

core/dbt/common/utils/__init__.py

@@ -0,0 +1,21 @@
+from dbt.common.utils.encoding import (
+    md5,
+    JSONEncoder,
+)
+
+from dbt.common.utils.casting import (
+    cast_to_str,
+    cast_to_int,
+    cast_dict_to_dict_of_strings,
+)
+
+from dbt.common.utils.dict import (
+    AttrDict,
+    filter_null_values,
+    merge,
+    deep_merge,
+    deep_merge_item,
+    deep_map_render,
+)
+
+from dbt.common.utils.executor import executor

core/dbt/common/utils/casting.py

@@ -0,0 +1,25 @@
+# This is useful for proto generated classes in particular, since
+# the default for protobuf for strings is the empty string, so
+# Optional[str] types don't work for generated Python classes.
+from typing import Optional
+
+
+def cast_to_str(string: Optional[str]) -> str:
+    if string is None:
+        return ""
+    else:
+        return string
+
+
+def cast_to_int(integer: Optional[int]) -> int:
+    if integer is None:
+        return 0
+    else:
+        return integer
+
+
+def cast_dict_to_dict_of_strings(dct):
+    new_dct = {}
+    for k, v in dct.items():
+        new_dct[str(k)] = str(v)
+    return new_dct

core/dbt/common/utils/dict.py

@@ -0,0 +1,128 @@
+import copy
+import datetime
+from typing import Dict, Optional, TypeVar, Callable, Any, Tuple, Union, Type
+
+from dbt.exceptions import DbtConfigError, RecursionError
+
+K_T = TypeVar("K_T")
+V_T = TypeVar("V_T")
+
+
+def filter_null_values(input: Dict[K_T, Optional[V_T]]) -> Dict[K_T, V_T]:
+    return {k: v for k, v in input.items() if v is not None}
+
+
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+def merge(*args):
+    if len(args) == 0:
+        return None
+
+    if len(args) == 1:
+        return args[0]
+
+    lst = list(args)
+    last = lst.pop(len(lst) - 1)
+
+    return _merge(merge(*lst), last)
+
+
+def _merge(a, b):
+    to_return = a.copy()
+    to_return.update(b)
+    return to_return
+
+
+# http://stackoverflow.com/questions/20656135/python-deep-merge-dictionary-data
+def deep_merge(*args):
+    """
+    >>> dbt.utils.deep_merge({'a': 1, 'b': 2, 'c': 3}, {'a': 2}, {'a': 3, 'b': 1})  # noqa
+    {'a': 3, 'b': 1, 'c': 3}
+    """
+    if len(args) == 0:
+        return None
+
+    if len(args) == 1:
+        return copy.deepcopy(args[0])
+
+    lst = list(args)
+    last = copy.deepcopy(lst.pop(len(lst) - 1))
+
+    return _deep_merge(deep_merge(*lst), last)
+
+
+def _deep_merge(destination, source):
+    if isinstance(source, dict):
+        for key, value in source.items():
+            deep_merge_item(destination, key, value)
+        return destination
+
+
+def deep_merge_item(destination, key, value):
+    if isinstance(value, dict):
+        node = destination.setdefault(key, {})
+        destination[key] = deep_merge(node, value)
+    elif isinstance(value, tuple) or isinstance(value, list):
+        if key in destination:
+            destination[key] = list(value) + list(destination[key])
+        else:
+            destination[key] = value
+    else:
+        destination[key] = value
+
+
+def _deep_map_render(
+    func: Callable[[Any, Tuple[Union[str, int], ...]], Any],
+    value: Any,
+    keypath: Tuple[Union[str, int], ...],
+) -> Any:
+    atomic_types: Tuple[Type[Any], ...] = (int, float, str, type(None), bool, datetime.date)
+
+    ret: Any
+
+    if isinstance(value, list):
+        ret = [_deep_map_render(func, v, (keypath + (idx,))) for idx, v in enumerate(value)]
+    elif isinstance(value, dict):
+        ret = {k: _deep_map_render(func, v, (keypath + (str(k),))) for k, v in value.items()}
+    elif isinstance(value, atomic_types):
+        ret = func(value, keypath)
+    else:
+        container_types: Tuple[Type[Any], ...] = (list, dict)
+        ok_types = container_types + atomic_types
+        raise DbtConfigError(
+            "in _deep_map_render, expected one of {!r}, got {!r}".format(ok_types, type(value))
+        )
+
+    return ret
+
+
+def deep_map_render(func: Callable[[Any, Tuple[Union[str, int], ...]], Any], value: Any) -> Any:
+    """This function renders a nested dictionary derived from a yaml
+    file. It is used to render dbt_project.yml, profiles.yml, and
+    schema files.
+
+    It maps the function func() onto each non-container value in 'value'
+    recursively, returning a new value. As long as func does not manipulate
+    the value, then deep_map_render will also not manipulate it.
+
+    value should be a value returned by `yaml.safe_load` or `json.load` - the
+    only expected types are list, dict, native python number, str, NoneType,
+    and bool.
+
+    func() will be called on numbers, strings, Nones, and booleans. Its first
+    parameter will be the value, and the second will be its keypath, an
+    iterable over the __getitem__ keys needed to get to it.
+
+    :raises: If there are cycles in the value, raises a
+        dbt.exceptions.RecursionException
+    """
+    try:
+        return _deep_map_render(func, value, ())
+    except RuntimeError as exc:
+        if "maximum recursion depth exceeded" in str(exc):
+            raise RecursionError("Cycle detected in deep_map_render")
+        raise

core/dbt/common/utils/encoding.py

@@ -0,0 +1,56 @@
+import datetime
+import decimal
+import hashlib
+import json
+from typing import Tuple, Type, Any
+
+import jinja2
+import sys
+
+DECIMALS: Tuple[Type[Any], ...]
+try:
+    import cdecimal  # typing: ignore
+except ImportError:
+    DECIMALS = (decimal.Decimal,)
+else:
+    DECIMALS = (decimal.Decimal, cdecimal.Decimal)
+
+
+def md5(string, charset="utf-8"):
+    if sys.version_info >= (3, 9):
+        return hashlib.md5(string.encode(charset), usedforsecurity=False).hexdigest()
+    else:
+        return hashlib.md5(string.encode(charset)).hexdigest()
+
+
+class JSONEncoder(json.JSONEncoder):
+    """A 'custom' json encoder that does normal json encoder things, but also
+    handles `Decimal`s and `Undefined`s. Decimals can lose precision because
+    they get converted to floats. Undefined's are serialized to an empty string
+    """
+
+    def default(self, obj):
+        if isinstance(obj, DECIMALS):
+            return float(obj)
+        elif isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
+            return obj.isoformat()
+        elif isinstance(obj, jinja2.Undefined):
+            return ""
+        elif isinstance(obj, Exception):
+            return repr(obj)
+        elif hasattr(obj, "to_dict"):
+            # if we have a to_dict we should try to serialize the result of
+            # that!
+            return obj.to_dict(omit_none=True)
+        else:
+            return super().default(obj)
+
+
+class ForgivingJSONEncoder(JSONEncoder):
+    def default(self, obj):
+        # let dbt's default JSON encoder handle it if possible, fallback to
+        # str()
+        try:
+            return super().default(obj)
+        except TypeError:
+            return str(obj)