Backports: networkx, context cleanup

.changes/unreleased/Dependencies-20220606-234258.yaml

@@ -0,0 +1,7 @@
+kind: Dependencies
+body: Pin networkx to <2.8.4 for v1.1 patches
+time: 2022-06-06T23:42:58.406625+02:00
+custom:
+  Author: jtcohen6
+  Issue: "5286"
+  PR: "5334"

.changes/unreleased/Under the Hood-20220606-230353.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: Update context readme + clean up context code"
+time: 2022-06-06T23:03:53.022568+02:00
+custom:
+  Author: jtcohen6
+  Issue: "4796"
+  PR: "5334"

core/dbt/config/renderer.py

@@ -1,12 +1,15 @@
 from typing import Dict, Any, Tuple, Optional, Union, Callable
+import re
+import os
 
 from dbt.clients.jinja import get_rendered, catch_jinja
 from dbt.context.target import TargetContext
-from dbt.context.secret import SecretContext
+from dbt.context.secret import SecretContext, SECRET_PLACEHOLDER
 from dbt.context.base import BaseContext
 from dbt.contracts.connection import HasCredentials
 from dbt.exceptions import DbtProjectError, CompilationException, RecursionException
 from dbt.utils import deep_map_render
+from dbt.logger import SECRET_ENV_PREFIX
 
 
 Keypath = Tuple[Union[str, int], ...]
@@ -174,6 +177,23 @@ def __init__(self, cli_vars: Dict[str, Any] = {}) -> None:
     def name(self):
         return "Secret"
 
+    def render_value(self, value: Any, keypath: Optional[Keypath] = None) -> Any:
+        rendered = super().render_value(value, keypath)
+        if SECRET_ENV_PREFIX in str(rendered):
+            search_group = f"({SECRET_ENV_PREFIX}(.*))"
+            pattern = SECRET_PLACEHOLDER.format(search_group).replace("$", r"\$")
+            m = re.search(
+                pattern,
+                rendered,
+            )
+            if m:
+                found = m.group(1)
+                value = os.environ[found]
+                replace_this = SECRET_PLACEHOLDER.format(found)
+                return rendered.replace(replace_this, value)
+        else:
+            return rendered
+
 
 class ProfileRenderer(SecretRenderer):
     @property

core/dbt/context/README.md

@@ -1 +1,51 @@
 # Contexts and Jinja rendering
+
+Contexts are used for Jinja rendering. They include context methods, executable macros, and various settings that are available in Jinja.
+
+The most common entrypoint to Jinja rendering in dbt is a method named `get_rendered`, which takes two arguments: templated code (string), and a context used to render it (dictionary). 
+
+The context is the bundle of information that is in "scope" when rendering Jinja-templated code. For instance, imagine a simple Jinja template:
+```
+{% set new_value = some_macro(some_variable) %}
+```
+Both `some_macro()` and `some_variable` must be defined in that context. Otherwise, it will raise an error when rendering.
+
+Different contexts are used in different places because we allow access to different methods and data in different places. Executable SQL, for example, includes all available macros and the model being run. The variables and macros in scope for Jinja defined in yaml files is much more limited.
+
+### Implementation
+
+The context that is passed to Jinja is always in a dictionary format, not an actual class, so a `to_dict()` is executed on a context class before it is used for rendering.
+
+Each context has a `generate_<name>_context` function to create the context. `ProviderContext` subclasses have different generate functions for parsing and for execution, so that certain functions (notably `ref`, `source`, and `config`) can return different results
+
+### Hierarchy
+
+All contexts inherit from the `BaseContext`, which includes "pure" methods (e.g. `tojson`), `env_var()`, and `var()` (but only CLI values, passed via `--vars`).
+
+Methods available in parent contexts are also available in child contexts.
+
+```
+   BaseContext -- core/dbt/context/base.py
+     SecretContext -- core/dbt/context/secret.py
+     TargetContext -- core/dbt/context/target.py
+       ConfiguredContext -- core/dbt/context/configured.py
+         SchemaYamlContext -- core/dbt/context/configured.py
+           DocsRuntimeContext -- core/dbt/context/configured.py
+         MacroResolvingContext -- core/dbt/context/configured.py
+         ManifestContext -- core/dbt/context/manifest.py
+           QueryHeaderContext -- core/dbt/context/manifest.py
+           ProviderContext -- core/dbt/context/provider.py
+             MacroContext -- core/dbt/context/provider.py
+             ModelContext -- core/dbt/context/provider.py
+             TestContext -- core/dbt/context/provider.py
+```
+
+### Contexts for configuration
+
+Contexts for rendering "special" `.yml` (configuration) files:
+- `SecretContext`: Supports "secret" env vars, which are prefixed with `DBT_ENV_SECRET_`. Used for rendering in `profiles.yml` and `packages.yml` ONLY. Secrets defined elsewhere will raise explicit errors.
+- `TargetContext`: The same as `Base`, plus `target` (connection profile). Used most notably in `dbt_project.yml` and `selectors.yml`.
+
+Contexts for other `.yml` files in the project:
+- `SchemaYamlContext`: Supports `vars` declared on the CLI and in `dbt_project.yml`. Does not support custom macros, beyond `var()` + `env_var()` methods. Used for all `.yml` files, to define properties and configuration.
+- `DocsRuntimeContext`: Standard `.yml` file context, plus `doc()` method (with all `docs` blocks in scope). Used to resolve `description` properties.

core/dbt/context/base.py

@@ -24,38 +24,7 @@
 import datetime
 import re
 
-# Contexts in dbt Core
-# Contexts are used for Jinja rendering. They include context methods,
-# executable macros, and various settings that are available in Jinja.
-#
-# Different contexts are used in different places because we allow access
-# to different methods and data in different places. Executable SQL, for
-# example, includes the available macros and the model, while Jinja in
-# yaml files is more limited.
-#
-# The context that is passed to Jinja is always in a dictionary format,
-# not an actual class, so a 'to_dict()' is executed on a context class
-# before it is used for rendering.
-#
-# Each context has a generate_<name>_context function to create the context.
-# ProviderContext subclasses have different generate functions for
-# parsing and for execution.
-#
-# Context class hierarchy
-#
-#   BaseContext -- core/dbt/context/base.py
-#     SecretContext -- core/dbt/context/secret.py
-#     TargetContext -- core/dbt/context/target.py
-#       ConfiguredContext -- core/dbt/context/configured.py
-#         SchemaYamlContext -- core/dbt/context/configured.py
-#           DocsRuntimeContext -- core/dbt/context/configured.py
-#         MacroResolvingContext -- core/dbt/context/configured.py
-#         ManifestContext -- core/dbt/context/manifest.py
-#           QueryHeaderContext -- core/dbt/context/manifest.py
-#           ProviderContext -- core/dbt/context/provider.py
-#             MacroContext -- core/dbt/context/provider.py
-#             ModelContext -- core/dbt/context/provider.py
-#             TestContext -- core/dbt/context/provider.py
+# See the `contexts` module README for more information on how contexts work
 
 
 def get_pytz_module_context() -> Dict[str, Any]:

core/dbt/context/secret.py

@@ -7,6 +7,9 @@
 from dbt.logger import SECRET_ENV_PREFIX
 
 
+SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
+
+
 class SecretContext(BaseContext):
     """This context is used in profiles.yml + packages.yml. It can render secret
     env vars that aren't usable elsewhere"""
@@ -18,21 +21,29 @@ def env_var(self, var: str, default: Optional[str] = None) -> str:
 
         If the default is None, raise an exception for an undefined variable.
 
-        In this context *only*, env_var will return the actual values of
-        env vars prefixed with DBT_ENV_SECRET_
+        In this context *only*, env_var will accept env vars prefixed with DBT_ENV_SECRET_.
+        It will return the name of the secret env var, wrapped in 'start' and 'end' identifiers.
+        The actual value will be subbed in later in SecretRenderer.render_value()
         """
         return_value = None
-        if var in os.environ:
+
+        # if this is a 'secret' env var, just return the name of the env var
+        # instead of rendering the actual value here, to avoid any risk of
+        # Jinja manipulation. it will be subbed out later, in SecretRenderer.render_value
+        if var in os.environ and var.startswith(SECRET_ENV_PREFIX):
+            return SECRET_PLACEHOLDER.format(var)
+
+        elif var in os.environ:
             return_value = os.environ[var]
         elif default is not None:
             return_value = default
 
         if return_value is not None:
-            # do not save secret environment variables
+            # store env vars in the internal manifest to power partial parsing
+            # if it's a 'secret' env var, we shouldn't even get here
+            # but just to be safe — don't save secrets
             if not var.startswith(SECRET_ENV_PREFIX):
                 self.env_vars[var] = return_value
-
-            # return the value even if its a secret
             return return_value
         else:
             msg = f"Env var required but not provided: '{var}'"

core/setup.py

@@ -61,7 +61,7 @@
         "logbook>=1.5,<1.6",
         "mashumaro==2.9",
         "minimal-snowplow-tracker==0.0.2",
-        "networkx>=2.3,<3",
+        "networkx>=2.3,<2.8.4",
         "packaging>=20.9,<22.0",
         "sqlparse>=0.2.3,<0.5",
         "dbt-extractor~=0.4.1",